sys/fs/cuse/cuse.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * Copyright (c) 2010-2013 Hans Petter Selasky. All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include "opt_compat.h"
  28
  29 #include <sys/stdint.h>
  30 #include <sys/stddef.h>
  31 #include <sys/param.h>
  32 #include <sys/types.h>
  33 #include <sys/systm.h>
  34 #include <sys/conf.h>
  35 #include <sys/kernel.h>
  36 #include <sys/bus.h>
  37 #include <sys/linker_set.h>
  38 #include <sys/module.h>
  39 #include <sys/lock.h>
  40 #include <sys/mutex.h>
  41 #include <sys/condvar.h>
  42 #include <sys/sysctl.h>
  43 #include <sys/unistd.h>
  44 #include <sys/malloc.h>
  45 #include <sys/priv.h>
  46 #include <sys/uio.h>
  47 #include <sys/poll.h>
  48 #include <sys/sx.h>
  49 #include <sys/queue.h>
  50 #include <sys/fcntl.h>
  51 #include <sys/proc.h>
  52 #include <sys/vnode.h>
  53 #include <sys/selinfo.h>
  54 #include <sys/ptrace.h>
  55
  56 #include <machine/bus.h>
  57
  58 #include <vm/vm.h>
  59 #include <vm/pmap.h>
  60
  61 #include <fs/cuse/cuse_defs.h>
  62 #include <fs/cuse/cuse_ioctl.h>
  63
  64 MODULE_VERSION(cuse, 1);
  65
  66 #define NBUSY   ((uint8_t *)1)
  67
  68 #ifdef FEATURE
  69 FEATURE(cuse, "Userspace character devices");
  70 #endif
  71
  72 struct cuse_command;
  73 struct cuse_server;
  74 struct cuse_client;
  75
  76 struct cuse_client_command {
  77         TAILQ_ENTRY(cuse_client_command) entry;
  78         struct cuse_command sub;
  79         struct sx sx;
  80         struct cv cv;
  81         struct thread *entered;
  82         struct cuse_client *client;
  83         struct proc *proc_curr;
  84         int     proc_refs;
  85         int     got_signal;
  86         int     error;
  87         int     command;
  88 };
  89
  90 struct cuse_memory {
  91         struct cuse_server *owner;
  92         uint8_t *virtaddr;
  93         uint32_t page_count;
  94         uint32_t is_allocated;
  95 };
  96
  97 struct cuse_server_dev {
  98         TAILQ_ENTRY(cuse_server_dev) entry;
  99         struct cuse_server *server;
 100         struct cdev *kern_dev;
 101         struct cuse_dev *user_dev;
 102 };
 103
 104 struct cuse_server {
 105         TAILQ_ENTRY(cuse_server) entry;
 106         TAILQ_HEAD(, cuse_client_command) head;
 107         TAILQ_HEAD(, cuse_server_dev) hdev;
 108         TAILQ_HEAD(, cuse_client) hcli;
 109         struct cv cv;
 110         struct selinfo selinfo;
 111         int     is_closing;
 112         int     refs;
 113 };
 114
 115 struct cuse_client {
 116         TAILQ_ENTRY(cuse_client) entry;
 117         TAILQ_ENTRY(cuse_client) entry_ref;
 118         struct cuse_client_command cmds[CUSE_CMD_MAX];
 119         struct cuse_server *server;
 120         struct cuse_server_dev *server_dev;
 121
 122         uint8_t ioctl_buffer[CUSE_BUFFER_MAX] __aligned(4);
 123
 124         int     fflags;         /* file flags */
 125         int     cflags;         /* client flags */
 126 #define CUSE_CLI_IS_CLOSING 0x01
 127 #define CUSE_CLI_KNOTE_NEED_READ 0x02
 128 #define CUSE_CLI_KNOTE_NEED_WRITE 0x04
 129 #define CUSE_CLI_KNOTE_HAS_READ 0x08
 130 #define CUSE_CLI_KNOTE_HAS_WRITE 0x10
 131 };
 132
 133 #define CUSE_CLIENT_CLOSING(pcc) \
 134     ((pcc)->cflags & CUSE_CLI_IS_CLOSING)
 135
 136 static MALLOC_DEFINE(M_CUSE, "cuse", "CUSE memory");
 137
 138 static TAILQ_HEAD(, cuse_server) cuse_server_head;
 139 static struct mtx cuse_mtx;
 140 static struct cdev *cuse_dev;
 141 static struct cuse_server *cuse_alloc_unit[CUSE_DEVICES_MAX];
 142 static int cuse_alloc_unit_id[CUSE_DEVICES_MAX];
 143 static struct cuse_memory cuse_mem[CUSE_ALLOC_UNIT_MAX];
 144
 145 static void cuse_client_kqfilter_read_detach(struct knote *kn);
 146 static void cuse_client_kqfilter_write_detach(struct knote *kn);
 147 static int cuse_client_kqfilter_read_event(struct knote *kn, long hint);
 148 static int cuse_client_kqfilter_write_event(struct knote *kn, long hint);
 149
 150 static struct filterops cuse_client_kqfilter_read_ops = {
 151         .f_isfd = 1,
 152         .f_detach = cuse_client_kqfilter_read_detach,
 153         .f_event = cuse_client_kqfilter_read_event,
 154 };
 155
 156 static struct filterops cuse_client_kqfilter_write_ops = {
 157         .f_isfd = 1,
 158         .f_detach = cuse_client_kqfilter_write_detach,
 159         .f_event = cuse_client_kqfilter_write_event,
 160 };
 161
 162 static d_open_t cuse_client_open;
 163 static d_close_t cuse_client_close;
 164 static d_ioctl_t cuse_client_ioctl;
 165 static d_read_t cuse_client_read;
 166 static d_write_t cuse_client_write;
 167 static d_poll_t cuse_client_poll;
 168 static d_mmap_t cuse_client_mmap;
 169 static d_kqfilter_t cuse_client_kqfilter;
 170
 171 static struct cdevsw cuse_client_devsw = {
 172         .d_version = D_VERSION,
 173         .d_open = cuse_client_open,
 174         .d_close = cuse_client_close,
 175         .d_ioctl = cuse_client_ioctl,
 176         .d_name = "cuse_client",
 177         .d_flags = D_TRACKCLOSE,
 178         .d_read = cuse_client_read,
 179         .d_write = cuse_client_write,
 180         .d_poll = cuse_client_poll,
 181         .d_mmap = cuse_client_mmap,
 182         .d_kqfilter = cuse_client_kqfilter,
 183 };
 184
 185 static d_open_t cuse_server_open;
 186 static d_close_t cuse_server_close;
 187 static d_ioctl_t cuse_server_ioctl;
 188 static d_read_t cuse_server_read;
 189 static d_write_t cuse_server_write;
 190 static d_poll_t cuse_server_poll;
 191 static d_mmap_t cuse_server_mmap;
 192
 193 static struct cdevsw cuse_server_devsw = {
 194         .d_version = D_VERSION,
 195         .d_open = cuse_server_open,
 196         .d_close = cuse_server_close,
 197         .d_ioctl = cuse_server_ioctl,
 198         .d_name = "cuse_server",
 199         .d_flags = D_TRACKCLOSE,
 200         .d_read = cuse_server_read,
 201         .d_write = cuse_server_write,
 202         .d_poll = cuse_server_poll,
 203         .d_mmap = cuse_server_mmap,
 204 };
 205
 206 static void cuse_client_is_closing(struct cuse_client *);
 207 static int cuse_free_unit_by_id_locked(struct cuse_server *, int);
 208
 209 static void
 210 cuse_lock(void)
 211 {
 212         mtx_lock(&cuse_mtx);
 213 }
 214
 215 static void
 216 cuse_unlock(void)
 217 {
 218         mtx_unlock(&cuse_mtx);
 219 }
 220
 221 static void
 222 cuse_cmd_lock(struct cuse_client_command *pccmd)
 223 {
 224         sx_xlock(&pccmd->sx);
 225 }
 226
 227 static void
 228 cuse_cmd_unlock(struct cuse_client_command *pccmd)
 229 {
 230         sx_xunlock(&pccmd->sx);
 231 }
 232
 233 static void
 234 cuse_kern_init(void *arg)
 235 {
 236         TAILQ_INIT(&cuse_server_head);
 237
 238         mtx_init(&cuse_mtx, "cuse-mtx", NULL, MTX_DEF);
 239
 240         cuse_dev = make_dev(&cuse_server_devsw, 0,
 241             UID_ROOT, GID_OPERATOR, 0600, "cuse");
 242
 243         printf("Cuse v%d.%d.%d @ /dev/cuse\n",
 244             (CUSE_VERSION >> 16) & 0xFF, (CUSE_VERSION >> 8) & 0xFF,
 245             (CUSE_VERSION >> 0) & 0xFF);
 246 }
 247
 248 SYSINIT(cuse_kern_init, SI_SUB_DEVFS, SI_ORDER_ANY, cuse_kern_init, 0);
 249
 250 static void
 251 cuse_kern_uninit(void *arg)
 252 {
 253         void *ptr;
 254
 255         while (1) {
 256
 257                 printf("Cuse: Please exit all /dev/cuse instances "
 258                     "and processes which have used this device.\n");
 259
 260                 pause("DRAIN", 2 * hz);
 261
 262                 cuse_lock();
 263                 ptr = TAILQ_FIRST(&cuse_server_head);
 264                 cuse_unlock();
 265
 266                 if (ptr == NULL)
 267                         break;
 268         }
 269
 270         if (cuse_dev != NULL)
 271                 destroy_dev(cuse_dev);
 272
 273         mtx_destroy(&cuse_mtx);
 274 }
 275
 276 SYSUNINIT(cuse_kern_uninit, SI_SUB_DEVFS, SI_ORDER_ANY, cuse_kern_uninit, 0);
 277
 278 static int
 279 cuse_server_get(struct cuse_server **ppcs)
 280 {
 281         struct cuse_server *pcs;
 282         int error;
 283
 284         error = devfs_get_cdevpriv((void **)&pcs);
 285         if (error != 0) {
 286                 *ppcs = NULL;
 287                 return (error);
 288         }
 289         /* check if closing */
 290         cuse_lock();
 291         if (pcs->is_closing) {
 292                 cuse_unlock();
 293                 *ppcs = NULL;
 294                 return (EINVAL);
 295         }
 296         cuse_unlock();
 297         *ppcs = pcs;
 298         return (0);
 299 }
 300
 301 static void
 302 cuse_server_is_closing(struct cuse_server *pcs)
 303 {
 304         struct cuse_client *pcc;
 305
 306         if (pcs->is_closing)
 307                 return;
 308
 309         pcs->is_closing = 1;
 310
 311         TAILQ_FOREACH(pcc, &pcs->hcli, entry) {
 312                 cuse_client_is_closing(pcc);
 313         }
 314 }
 315
 316 static struct cuse_client_command *
 317 cuse_server_find_command(struct cuse_server *pcs, struct thread *td)
 318 {
 319         struct cuse_client *pcc;
 320         int n;
 321
 322         if (pcs->is_closing)
 323                 goto done;
 324
 325         TAILQ_FOREACH(pcc, &pcs->hcli, entry) {
 326                 if (CUSE_CLIENT_CLOSING(pcc))
 327                         continue;
 328                 for (n = 0; n != CUSE_CMD_MAX; n++) {
 329                         if (pcc->cmds[n].entered == td)
 330                                 return (&pcc->cmds[n]);
 331                 }
 332         }
 333 done:
 334         return (NULL);
 335 }
 336
 337 static void
 338 cuse_str_filter(char *ptr)
 339 {
 340         int c;
 341
 342         while (((c = *ptr) != 0)) {
 343
 344                 if ((c >= 'a') && (c <= 'z')) {
 345                         ptr++;
 346                         continue;
 347                 }
 348                 if ((c >= 'A') && (c <= 'Z')) {
 349                         ptr++;
 350                         continue;
 351                 }
 352                 if ((c >= '0') && (c <= '9')) {
 353                         ptr++;
 354                         continue;
 355                 }
 356                 if ((c == '.') || (c == '_') || (c == '/')) {
 357                         ptr++;
 358                         continue;
 359                 }
 360                 *ptr = '_';
 361
 362                 ptr++;
 363         }
 364 }
 365
 366 static int
 367 cuse_convert_error(int error)
 368 {
 369         ;                               /* indent fix */
 370         switch (error) {
 371         case CUSE_ERR_NONE:
 372                 return (0);
 373         case CUSE_ERR_BUSY:
 374                 return (EBUSY);
 375         case CUSE_ERR_WOULDBLOCK:
 376                 return (EWOULDBLOCK);
 377         case CUSE_ERR_INVALID:
 378                 return (EINVAL);
 379         case CUSE_ERR_NO_MEMORY:
 380                 return (ENOMEM);
 381         case CUSE_ERR_FAULT:
 382                 return (EFAULT);
 383         case CUSE_ERR_SIGNAL:
 384                 return (EINTR);
 385         default:
 386                 return (ENXIO);
 387         }
 388 }
 389
 390 static void
 391 cuse_server_free_memory(struct cuse_server *pcs)
 392 {
 393         struct cuse_memory *mem;
 394         uint32_t n;
 395
 396         for (n = 0; n != CUSE_ALLOC_UNIT_MAX; n++) {
 397                 mem = &cuse_mem[n];
 398
 399                 /* this memory is never freed */
 400                 if (mem->owner == pcs) {
 401                         mem->owner = NULL;
 402                         mem->is_allocated = 0;
 403                 }
 404         }
 405 }
 406
 407 static int
 408 cuse_server_alloc_memory(struct cuse_server *pcs,
 409     struct cuse_memory *mem, uint32_t page_count)
 410 {
 411         void *ptr;
 412         int error;
 413
 414         cuse_lock();
 415
 416         if (mem->virtaddr == NBUSY) {
 417                 cuse_unlock();
 418                 return (EBUSY);
 419         }
 420         if (mem->virtaddr != NULL) {
 421                 if (mem->is_allocated != 0) {
 422                         cuse_unlock();
 423                         return (EBUSY);
 424                 }
 425                 if (mem->page_count == page_count) {
 426                         mem->is_allocated = 1;
 427                         mem->owner = pcs;
 428                         cuse_unlock();
 429                         return (0);
 430                 }
 431                 cuse_unlock();
 432                 return (EBUSY);
 433         }
 434         memset(mem, 0, sizeof(*mem));
 435
 436         mem->virtaddr = NBUSY;
 437
 438         cuse_unlock();
 439
 440         ptr = malloc(page_count * PAGE_SIZE, M_CUSE, M_WAITOK | M_ZERO);
 441         if (ptr == NULL)
 442                 error = ENOMEM;
 443         else
 444                 error = 0;
 445
 446         cuse_lock();
 447
 448         if (error) {
 449                 mem->virtaddr = NULL;
 450                 cuse_unlock();
 451                 return (error);
 452         }
 453         mem->virtaddr = ptr;
 454         mem->page_count = page_count;
 455         mem->is_allocated = 1;
 456         mem->owner = pcs;
 457         cuse_unlock();
 458
 459         return (0);
 460 }
 461
 462 static int
 463 cuse_client_get(struct cuse_client **ppcc)
 464 {
 465         struct cuse_client *pcc;
 466         int error;
 467
 468         /* try to get private data */
 469         error = devfs_get_cdevpriv((void **)&pcc);
 470         if (error != 0) {
 471                 *ppcc = NULL;
 472                 return (error);
 473         }
 474         /* check if closing */
 475         cuse_lock();
 476         if (CUSE_CLIENT_CLOSING(pcc) || pcc->server->is_closing) {
 477                 cuse_unlock();
 478                 *ppcc = NULL;
 479                 return (EINVAL);
 480         }
 481         cuse_unlock();
 482         *ppcc = pcc;
 483         return (0);
 484 }
 485
 486 static void
 487 cuse_client_is_closing(struct cuse_client *pcc)
 488 {
 489         struct cuse_client_command *pccmd;
 490         uint32_t n;
 491
 492         if (CUSE_CLIENT_CLOSING(pcc))
 493                 return;
 494
 495         pcc->cflags |= CUSE_CLI_IS_CLOSING;
 496         pcc->server_dev = NULL;
 497
 498         for (n = 0; n != CUSE_CMD_MAX; n++) {
 499
 500                 pccmd = &pcc->cmds[n];
 501
 502                 if (pccmd->entry.tqe_prev != NULL) {
 503                         TAILQ_REMOVE(&pcc->server->head, pccmd, entry);
 504                         pccmd->entry.tqe_prev = NULL;
 505                 }
 506                 cv_broadcast(&pccmd->cv);
 507         }
 508 }
 509
 510 static void
 511 cuse_client_send_command_locked(struct cuse_client_command *pccmd,
 512     unsigned long data_ptr, unsigned long arg, int fflags, int ioflag)
 513 {
 514         unsigned long cuse_fflags = 0;
 515         struct cuse_server *pcs;
 516
 517         if (fflags & FREAD)
 518                 cuse_fflags |= CUSE_FFLAG_READ;
 519
 520         if (fflags & FWRITE)
 521                 cuse_fflags |= CUSE_FFLAG_WRITE;
 522
 523         if (ioflag & IO_NDELAY)
 524                 cuse_fflags |= CUSE_FFLAG_NONBLOCK;
 525
 526         pccmd->sub.fflags = cuse_fflags;
 527         pccmd->sub.data_pointer = data_ptr;
 528         pccmd->sub.argument = arg;
 529
 530         pcs = pccmd->client->server;
 531
 532         if ((pccmd->entry.tqe_prev == NULL) &&
 533             (CUSE_CLIENT_CLOSING(pccmd->client) == 0) &&
 534             (pcs->is_closing == 0)) {
 535                 TAILQ_INSERT_TAIL(&pcs->head, pccmd, entry);
 536                 cv_signal(&pcs->cv);
 537         }
 538 }
 539
 540 static void
 541 cuse_client_got_signal(struct cuse_client_command *pccmd)
 542 {
 543         struct cuse_server *pcs;
 544
 545         pccmd->got_signal = 1;
 546
 547         pccmd = &pccmd->client->cmds[CUSE_CMD_SIGNAL];
 548
 549         pcs = pccmd->client->server;
 550
 551         if ((pccmd->entry.tqe_prev == NULL) &&
 552             (CUSE_CLIENT_CLOSING(pccmd->client) == 0) &&
 553             (pcs->is_closing == 0)) {
 554                 TAILQ_INSERT_TAIL(&pcs->head, pccmd, entry);
 555                 cv_signal(&pcs->cv);
 556         }
 557 }
 558
 559 static int
 560 cuse_client_receive_command_locked(struct cuse_client_command *pccmd,
 561     uint8_t *arg_ptr, uint32_t arg_len)
 562 {
 563         int error;
 564
 565         error = 0;
 566
 567         pccmd->proc_curr = curthread->td_proc;
 568
 569         if (CUSE_CLIENT_CLOSING(pccmd->client) ||
 570             pccmd->client->server->is_closing) {
 571                 error = CUSE_ERR_OTHER;
 572                 goto done;
 573         }
 574         while (pccmd->command == CUSE_CMD_NONE) {
 575                 if (error != 0) {
 576                         cv_wait(&pccmd->cv, &cuse_mtx);
 577                 } else {
 578                         error = cv_wait_sig(&pccmd->cv, &cuse_mtx);
 579
 580                         if (error != 0)
 581                                 cuse_client_got_signal(pccmd);
 582                 }
 583                 if (CUSE_CLIENT_CLOSING(pccmd->client) ||
 584                     pccmd->client->server->is_closing) {
 585                         error = CUSE_ERR_OTHER;
 586                         goto done;
 587                 }
 588         }
 589
 590         error = pccmd->error;
 591         pccmd->command = CUSE_CMD_NONE;
 592         cv_signal(&pccmd->cv);
 593
 594 done:
 595
 596         /* wait until all process references are gone */
 597
 598         pccmd->proc_curr = NULL;
 599
 600         while (pccmd->proc_refs != 0)
 601                 cv_wait(&pccmd->cv, &cuse_mtx);
 602
 603         return (error);
 604 }
 605
 606 /*------------------------------------------------------------------------*
 607  *      CUSE SERVER PART
 608  *------------------------------------------------------------------------*/
 609
 610 static void
 611 cuse_server_free_dev(struct cuse_server_dev *pcsd)
 612 {
 613         struct cuse_server *pcs;
 614         struct cuse_client *pcc;
 615
 616         /* get server pointer */
 617         pcs = pcsd->server;
 618
 619         /* prevent creation of more devices */
 620         cuse_lock();
 621         if (pcsd->kern_dev != NULL)
 622                 pcsd->kern_dev->si_drv1 = NULL;
 623
 624         TAILQ_FOREACH(pcc, &pcs->hcli, entry) {
 625                 if (pcc->server_dev == pcsd)
 626                         cuse_client_is_closing(pcc);
 627         }
 628         cuse_unlock();
 629
 630         /* destroy device, if any */
 631         if (pcsd->kern_dev != NULL) {
 632                 /* destroy device synchronously */
 633                 destroy_dev(pcsd->kern_dev);
 634         }
 635         free(pcsd, M_CUSE);
 636 }
 637
 638 static void
 639 cuse_server_free(void *arg)
 640 {
 641         struct cuse_server *pcs = arg;
 642         struct cuse_server_dev *pcsd;
 643
 644         cuse_lock();
 645         pcs->refs--;
 646         if (pcs->refs != 0) {
 647                 cuse_unlock();
 648                 return;
 649         }
 650         cuse_server_is_closing(pcs);
 651
 652         TAILQ_REMOVE(&cuse_server_head, pcs, entry);
 653
 654         cuse_free_unit_by_id_locked(pcs, -1);
 655
 656         while ((pcsd = TAILQ_FIRST(&pcs->hdev)) != NULL) {
 657                 TAILQ_REMOVE(&pcs->hdev, pcsd, entry);
 658                 cuse_unlock();
 659                 cuse_server_free_dev(pcsd);
 660                 cuse_lock();
 661         }
 662
 663         cuse_server_free_memory(pcs);
 664
 665         knlist_clear(&pcs->selinfo.si_note, 1);
 666         knlist_destroy(&pcs->selinfo.si_note);
 667
 668         cuse_unlock();
 669
 670         seldrain(&pcs->selinfo);
 671
 672         cv_destroy(&pcs->cv);
 673
 674         free(pcs, M_CUSE);
 675 }
 676
 677 static int
 678 cuse_server_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
 679 {
 680         struct cuse_server *pcs;
 681
 682         pcs = malloc(sizeof(*pcs), M_CUSE, M_WAITOK | M_ZERO);
 683         if (pcs == NULL)
 684                 return (ENOMEM);
 685
 686         if (devfs_set_cdevpriv(pcs, &cuse_server_free)) {
 687                 printf("Cuse: Cannot set cdevpriv.\n");
 688                 free(pcs, M_CUSE);
 689                 return (ENOMEM);
 690         }
 691         TAILQ_INIT(&pcs->head);
 692         TAILQ_INIT(&pcs->hdev);
 693         TAILQ_INIT(&pcs->hcli);
 694
 695         cv_init(&pcs->cv, "cuse-server-cv");
 696
 697         knlist_init_mtx(&pcs->selinfo.si_note, &cuse_mtx);
 698
 699         cuse_lock();
 700         pcs->refs++;
 701         TAILQ_INSERT_TAIL(&cuse_server_head, pcs, entry);
 702         cuse_unlock();
 703
 704         return (0);
 705 }
 706
 707 static int
 708 cuse_server_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
 709 {
 710         struct cuse_server *pcs;
 711         int error;
 712
 713         error = cuse_server_get(&pcs);
 714         if (error != 0)
 715                 goto done;
 716
 717         cuse_lock();
 718         cuse_server_is_closing(pcs);
 719         knlist_clear(&pcs->selinfo.si_note, 1);
 720         cuse_unlock();
 721
 722 done:
 723         return (0);
 724 }
 725
 726 static int
 727 cuse_server_read(struct cdev *dev, struct uio *uio, int ioflag)
 728 {
 729         return (ENXIO);
 730 }
 731
 732 static int
 733 cuse_server_write(struct cdev *dev, struct uio *uio, int ioflag)
 734 {
 735         return (ENXIO);
 736 }
 737
 738 static int
 739 cuse_server_ioctl_copy_locked(struct cuse_client_command *pccmd,
 740     struct cuse_data_chunk *pchk, int isread)
 741 {
 742         struct proc *p_proc;
 743         uint32_t offset;
 744         int error;
 745
 746         offset = pchk->peer_ptr - CUSE_BUF_MIN_PTR;
 747
 748         if (pchk->length > CUSE_BUFFER_MAX)
 749                 return (EFAULT);
 750
 751         if (offset >= CUSE_BUFFER_MAX)
 752                 return (EFAULT);
 753
 754         if ((offset + pchk->length) > CUSE_BUFFER_MAX)
 755                 return (EFAULT);
 756
 757         p_proc = pccmd->proc_curr;
 758         if (p_proc == NULL)
 759                 return (ENXIO);
 760
 761         if (pccmd->proc_refs < 0)
 762                 return (ENOMEM);
 763
 764         pccmd->proc_refs++;
 765
 766         cuse_unlock();
 767
 768         if (isread == 0) {
 769                 error = copyin(
 770                     (void *)pchk->local_ptr,
 771                     pccmd->client->ioctl_buffer + offset,
 772                     pchk->length);
 773         } else {
 774                 error = copyout(
 775                     pccmd->client->ioctl_buffer + offset,
 776                     (void *)pchk->local_ptr,
 777                     pchk->length);
 778         }
 779
 780         cuse_lock();
 781
 782         pccmd->proc_refs--;
 783
 784         if (pccmd->proc_curr == NULL)
 785                 cv_signal(&pccmd->cv);
 786
 787         return (error);
 788 }
 789
 790 static int
 791 cuse_proc2proc_copy(struct proc *proc_s, vm_offset_t data_s,
 792     struct proc *proc_d, vm_offset_t data_d, size_t len)
 793 {
 794         struct thread *td;
 795         struct proc *proc_cur;
 796         int error;
 797
 798         td = curthread;
 799         proc_cur = td->td_proc;
 800
 801         if (proc_cur == proc_d) {
 802                 struct iovec iov = {
 803                         .iov_base = (caddr_t)data_d,
 804                         .iov_len = len,
 805                 };
 806                 struct uio uio = {
 807                         .uio_iov = &iov,
 808                         .uio_iovcnt = 1,
 809                         .uio_offset = (off_t)data_s,
 810                         .uio_resid = len,
 811                         .uio_segflg = UIO_USERSPACE,
 812                         .uio_rw = UIO_READ,
 813                         .uio_td = td,
 814                 };
 815
 816                 PHOLD(proc_s);
 817                 error = proc_rwmem(proc_s, &uio);
 818                 PRELE(proc_s);
 819
 820         } else if (proc_cur == proc_s) {
 821                 struct iovec iov = {
 822                         .iov_base = (caddr_t)data_s,
 823                         .iov_len = len,
 824                 };
 825                 struct uio uio = {
 826                         .uio_iov = &iov,
 827                         .uio_iovcnt = 1,
 828                         .uio_offset = (off_t)data_d,
 829                         .uio_resid = len,
 830                         .uio_segflg = UIO_USERSPACE,
 831                         .uio_rw = UIO_WRITE,
 832                         .uio_td = td,
 833                 };
 834
 835                 PHOLD(proc_d);
 836                 error = proc_rwmem(proc_d, &uio);
 837                 PRELE(proc_d);
 838         } else {
 839                 error = EINVAL;
 840         }
 841         return (error);
 842 }
 843
 844 static int
 845 cuse_server_data_copy_locked(struct cuse_client_command *pccmd,
 846     struct cuse_data_chunk *pchk, int isread)
 847 {
 848         struct proc *p_proc;
 849         int error;
 850
 851         p_proc = pccmd->proc_curr;
 852         if (p_proc == NULL)
 853                 return (ENXIO);
 854
 855         if (pccmd->proc_refs < 0)
 856                 return (ENOMEM);
 857
 858         pccmd->proc_refs++;
 859
 860         cuse_unlock();
 861
 862         if (isread == 0) {
 863                 error = cuse_proc2proc_copy(
 864                     curthread->td_proc, pchk->local_ptr,
 865                     p_proc, pchk->peer_ptr,
 866                     pchk->length);
 867         } else {
 868                 error = cuse_proc2proc_copy(
 869                     p_proc, pchk->peer_ptr,
 870                     curthread->td_proc, pchk->local_ptr,
 871                     pchk->length);
 872         }
 873
 874         cuse_lock();
 875
 876         pccmd->proc_refs--;
 877
 878         if (pccmd->proc_curr == NULL)
 879                 cv_signal(&pccmd->cv);
 880
 881         return (error);
 882 }
 883
 884 static int
 885 cuse_alloc_unit_by_id_locked(struct cuse_server *pcs, int id)
 886 {
 887         int n;
 888         int x = 0;
 889         int match;
 890
 891         do {
 892                 for (match = n = 0; n != CUSE_DEVICES_MAX; n++) {
 893                         if (cuse_alloc_unit[n] != NULL) {
 894                                 if ((cuse_alloc_unit_id[n] ^ id) & CUSE_ID_MASK)
 895                                         continue;
 896                                 if ((cuse_alloc_unit_id[n] & ~CUSE_ID_MASK) == x) {
 897                                         x++;
 898                                         match = 1;
 899                                 }
 900                         }
 901                 }
 902         } while (match);
 903
 904         if (x < 256) {
 905                 for (n = 0; n != CUSE_DEVICES_MAX; n++) {
 906                         if (cuse_alloc_unit[n] == NULL) {
 907                                 cuse_alloc_unit[n] = pcs;
 908                                 cuse_alloc_unit_id[n] = id | x;
 909                                 return (x);
 910                         }
 911                 }
 912         }
 913         return (-1);
 914 }
 915
 916 static void
 917 cuse_server_wakeup_locked(struct cuse_server *pcs)
 918 {
 919         selwakeup(&pcs->selinfo);
 920         KNOTE_LOCKED(&pcs->selinfo.si_note, 0);
 921 }
 922
 923 static int
 924 cuse_free_unit_by_id_locked(struct cuse_server *pcs, int id)
 925 {
 926         int n;
 927         int found = 0;
 928
 929         for (n = 0; n != CUSE_DEVICES_MAX; n++) {
 930                 if (cuse_alloc_unit[n] == pcs) {
 931                         if (cuse_alloc_unit_id[n] == id || id == -1) {
 932                                 cuse_alloc_unit[n] = NULL;
 933                                 cuse_alloc_unit_id[n] = 0;
 934                                 found = 1;
 935                         }
 936                 }
 937         }
 938
 939         return (found ? 0 : EINVAL);
 940 }
 941
 942 static int
 943 cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
 944     caddr_t data, int fflag, struct thread *td)
 945 {
 946         struct cuse_server *pcs;
 947         int error;
 948
 949         error = cuse_server_get(&pcs);
 950         if (error != 0)
 951                 return (error);
 952
 953         switch (cmd) {
 954                 struct cuse_client_command *pccmd;
 955                 struct cuse_client *pcc;
 956                 struct cuse_command *pcmd;
 957                 struct cuse_alloc_info *pai;
 958                 struct cuse_create_dev *pcd;
 959                 struct cuse_server_dev *pcsd;
 960                 struct cuse_data_chunk *pchk;
 961                 int n;
 962
 963         case CUSE_IOCTL_GET_COMMAND:
 964                 pcmd = (void *)data;
 965
 966                 cuse_lock();
 967
 968                 while ((pccmd = TAILQ_FIRST(&pcs->head)) == NULL) {
 969                         error = cv_wait_sig(&pcs->cv, &cuse_mtx);
 970
 971                         if (pcs->is_closing)
 972                                 error = ENXIO;
 973
 974                         if (error) {
 975                                 cuse_unlock();
 976                                 return (error);
 977                         }
 978                 }
 979
 980                 TAILQ_REMOVE(&pcs->head, pccmd, entry);
 981                 pccmd->entry.tqe_prev = NULL;
 982
 983                 pccmd->entered = curthread;
 984
 985                 *pcmd = pccmd->sub;
 986
 987                 cuse_unlock();
 988
 989                 break;
 990
 991         case CUSE_IOCTL_SYNC_COMMAND:
 992
 993                 cuse_lock();
 994                 while ((pccmd = cuse_server_find_command(pcs, curthread)) != NULL) {
 995
 996                         /* send sync command */
 997                         pccmd->entered = NULL;
 998                         pccmd->error = *(int *)data;
 999                         pccmd->command = CUSE_CMD_SYNC;
1000
1001                         /* signal peer, if any */
1002                         cv_signal(&pccmd->cv);
1003                 }
1004                 cuse_unlock();
1005
1006                 break;
1007
1008         case CUSE_IOCTL_ALLOC_UNIT:
1009
1010                 cuse_lock();
1011                 n = cuse_alloc_unit_by_id_locked(pcs,
1012                     CUSE_ID_DEFAULT(0));
1013                 cuse_unlock();
1014
1015                 if (n < 0)
1016                         error = ENOMEM;
1017                 else
1018                         *(int *)data = n;
1019                 break;
1020
1021         case CUSE_IOCTL_ALLOC_UNIT_BY_ID:
1022
1023                 n = *(int *)data;
1024
1025                 n = (n & CUSE_ID_MASK);
1026
1027                 cuse_lock();
1028                 n = cuse_alloc_unit_by_id_locked(pcs, n);
1029                 cuse_unlock();
1030
1031                 if (n < 0)
1032                         error = ENOMEM;
1033                 else
1034                         *(int *)data = n;
1035                 break;
1036
1037         case CUSE_IOCTL_FREE_UNIT:
1038
1039                 n = *(int *)data;
1040
1041                 n = CUSE_ID_DEFAULT(n);
1042
1043                 cuse_lock();
1044                 error = cuse_free_unit_by_id_locked(pcs, n);
1045                 cuse_unlock();
1046                 break;
1047
1048         case CUSE_IOCTL_FREE_UNIT_BY_ID:
1049
1050                 n = *(int *)data;
1051
1052                 cuse_lock();
1053                 error = cuse_free_unit_by_id_locked(pcs, n);
1054                 cuse_unlock();
1055                 break;
1056
1057         case CUSE_IOCTL_ALLOC_MEMORY:
1058
1059                 pai = (void *)data;
1060
1061                 if (pai->alloc_nr >= CUSE_ALLOC_UNIT_MAX) {
1062                         error = ENOMEM;
1063                         break;
1064                 }
1065                 if (pai->page_count > CUSE_ALLOC_PAGES_MAX) {
1066                         error = ENOMEM;
1067                         break;
1068                 }
1069                 error = cuse_server_alloc_memory(pcs,
1070                     &cuse_mem[pai->alloc_nr], pai->page_count);
1071                 break;
1072
1073         case CUSE_IOCTL_FREE_MEMORY:
1074                 pai = (void *)data;
1075
1076                 if (pai->alloc_nr >= CUSE_ALLOC_UNIT_MAX) {
1077                         error = ENOMEM;
1078                         break;
1079                 }
1080                 /* we trust the character device driver in this case */
1081
1082                 cuse_lock();
1083                 if (cuse_mem[pai->alloc_nr].owner == pcs) {
1084                         cuse_mem[pai->alloc_nr].is_allocated = 0;
1085                         cuse_mem[pai->alloc_nr].owner = NULL;
1086                 } else {
1087                         error = EINVAL;
1088                 }
1089                 cuse_unlock();
1090                 break;
1091
1092         case CUSE_IOCTL_GET_SIG:
1093
1094                 cuse_lock();
1095                 pccmd = cuse_server_find_command(pcs, curthread);
1096
1097                 if (pccmd != NULL) {
1098                         n = pccmd->got_signal;
1099                         pccmd->got_signal = 0;
1100                 } else {
1101                         n = 0;
1102                 }
1103                 cuse_unlock();
1104
1105                 *(int *)data = n;
1106
1107                 break;
1108
1109         case CUSE_IOCTL_SET_PFH:
1110
1111                 cuse_lock();
1112                 pccmd = cuse_server_find_command(pcs, curthread);
1113
1114                 if (pccmd != NULL) {
1115                         pcc = pccmd->client;
1116                         for (n = 0; n != CUSE_CMD_MAX; n++) {
1117                                 pcc->cmds[n].sub.per_file_handle = *(unsigned long *)data;
1118                         }
1119                 } else {
1120                         error = ENXIO;
1121                 }
1122                 cuse_unlock();
1123                 break;
1124
1125         case CUSE_IOCTL_CREATE_DEV:
1126
1127                 error = priv_check(curthread, PRIV_DRIVER);
1128                 if (error)
1129                         break;
1130
1131                 pcd = (void *)data;
1132
1133                 /* filter input */
1134
1135                 pcd->devname[sizeof(pcd->devname) - 1] = 0;
1136
1137                 if (pcd->devname[0] == 0) {
1138                         error = EINVAL;
1139                         break;
1140                 }
1141                 cuse_str_filter(pcd->devname);
1142
1143                 pcd->permissions &= 0777;
1144
1145                 /* try to allocate a character device */
1146
1147                 pcsd = malloc(sizeof(*pcsd), M_CUSE, M_WAITOK | M_ZERO);
1148
1149                 if (pcsd == NULL) {
1150                         error = ENOMEM;
1151                         break;
1152                 }
1153                 pcsd->server = pcs;
1154
1155                 pcsd->user_dev = pcd->dev;
1156
1157                 pcsd->kern_dev = make_dev_credf(MAKEDEV_CHECKNAME,
1158                     &cuse_client_devsw, 0, NULL, pcd->user_id, pcd->group_id,
1159                     pcd->permissions, "%s", pcd->devname);
1160
1161                 if (pcsd->kern_dev == NULL) {
1162                         free(pcsd, M_CUSE);
1163                         error = ENOMEM;
1164                         break;
1165                 }
1166                 pcsd->kern_dev->si_drv1 = pcsd;
1167
1168                 cuse_lock();
1169                 TAILQ_INSERT_TAIL(&pcs->hdev, pcsd, entry);
1170                 cuse_unlock();
1171
1172                 break;
1173
1174         case CUSE_IOCTL_DESTROY_DEV:
1175
1176                 error = priv_check(curthread, PRIV_DRIVER);
1177                 if (error)
1178                         break;
1179
1180                 cuse_lock();
1181
1182                 error = EINVAL;
1183
1184                 pcsd = TAILQ_FIRST(&pcs->hdev);
1185                 while (pcsd != NULL) {
1186                         if (pcsd->user_dev == *(struct cuse_dev **)data) {
1187                                 TAILQ_REMOVE(&pcs->hdev, pcsd, entry);
1188                                 cuse_unlock();
1189                                 cuse_server_free_dev(pcsd);
1190                                 cuse_lock();
1191                                 error = 0;
1192                                 pcsd = TAILQ_FIRST(&pcs->hdev);
1193                         } else {
1194                                 pcsd = TAILQ_NEXT(pcsd, entry);
1195                         }
1196                 }
1197
1198                 cuse_unlock();
1199                 break;
1200
1201         case CUSE_IOCTL_WRITE_DATA:
1202         case CUSE_IOCTL_READ_DATA:
1203
1204                 cuse_lock();
1205                 pchk = (struct cuse_data_chunk *)data;
1206
1207                 pccmd = cuse_server_find_command(pcs, curthread);
1208
1209                 if (pccmd == NULL) {
1210                         error = ENXIO;  /* invalid request */
1211                 } else if (pchk->peer_ptr < CUSE_BUF_MIN_PTR) {
1212                         error = EFAULT; /* NULL pointer */
1213                 } else if (pchk->peer_ptr < CUSE_BUF_MAX_PTR) {
1214                         error = cuse_server_ioctl_copy_locked(pccmd,
1215                             pchk, cmd == CUSE_IOCTL_READ_DATA);
1216                 } else {
1217                         error = cuse_server_data_copy_locked(pccmd,
1218                             pchk, cmd == CUSE_IOCTL_READ_DATA);
1219                 }
1220                 cuse_unlock();
1221                 break;
1222
1223         case CUSE_IOCTL_SELWAKEUP:
1224                 cuse_lock();
1225                 /*
1226                  * We don't know which direction caused the event.
1227                  * Wakeup both!
1228                  */
1229                 TAILQ_FOREACH(pcc, &pcs->hcli, entry) {
1230                         pcc->cflags |= (CUSE_CLI_KNOTE_NEED_READ |
1231                             CUSE_CLI_KNOTE_NEED_WRITE);
1232                 }
1233                 cuse_server_wakeup_locked(pcs);
1234                 cuse_unlock();
1235                 break;
1236
1237         default:
1238                 error = ENXIO;
1239                 break;
1240         }
1241         return (error);
1242 }
1243
1244 static int
1245 cuse_server_poll(struct cdev *dev, int events, struct thread *td)
1246 {
1247         return (events & (POLLHUP | POLLPRI | POLLIN |
1248             POLLRDNORM | POLLOUT | POLLWRNORM));
1249 }
1250
1251 static int
1252 cuse_server_mmap(struct cdev *dev, vm_ooffset_t offset, vm_paddr_t *paddr, int nprot, vm_memattr_t *memattr)
1253 {
1254         uint32_t page_nr = offset / PAGE_SIZE;
1255         uint32_t alloc_nr = page_nr / CUSE_ALLOC_PAGES_MAX;
1256         struct cuse_memory *mem;
1257         struct cuse_server *pcs;
1258         uint8_t *ptr;
1259         int error;
1260
1261         if (alloc_nr >= CUSE_ALLOC_UNIT_MAX)
1262                 return (ENOMEM);
1263
1264         error = cuse_server_get(&pcs);
1265         if (error != 0)
1266                 pcs = NULL;
1267
1268         cuse_lock();
1269         mem = &cuse_mem[alloc_nr];
1270
1271         /* try to enforce slight ownership */
1272         if ((pcs != NULL) && (mem->owner != pcs)) {
1273                 cuse_unlock();
1274                 return (EINVAL);
1275         }
1276         if (mem->virtaddr == NULL) {
1277                 cuse_unlock();
1278                 return (ENOMEM);
1279         }
1280         if (mem->virtaddr == NBUSY) {
1281                 cuse_unlock();
1282                 return (ENOMEM);
1283         }
1284         page_nr %= CUSE_ALLOC_PAGES_MAX;
1285
1286         if (page_nr >= mem->page_count) {
1287                 cuse_unlock();
1288                 return (ENXIO);
1289         }
1290         ptr = mem->virtaddr + (page_nr * PAGE_SIZE);
1291         cuse_unlock();
1292
1293         *paddr = vtophys(ptr);
1294
1295         return (0);
1296 }
1297
1298 /*------------------------------------------------------------------------*
1299  *      CUSE CLIENT PART
1300  *------------------------------------------------------------------------*/
1301 static void
1302 cuse_client_free(void *arg)
1303 {
1304         struct cuse_client *pcc = arg;
1305         struct cuse_client_command *pccmd;
1306         struct cuse_server *pcs;
1307         int n;
1308
1309         cuse_lock();
1310         cuse_client_is_closing(pcc);
1311         TAILQ_REMOVE(&pcc->server->hcli, pcc, entry);
1312         cuse_unlock();
1313
1314         for (n = 0; n != CUSE_CMD_MAX; n++) {
1315
1316                 pccmd = &pcc->cmds[n];
1317
1318                 sx_destroy(&pccmd->sx);
1319                 cv_destroy(&pccmd->cv);
1320         }
1321
1322         pcs = pcc->server;
1323
1324         free(pcc, M_CUSE);
1325
1326         /* drop reference on server */
1327         cuse_server_free(pcs);
1328 }
1329
1330 static int
1331 cuse_client_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
1332 {
1333         struct cuse_client_command *pccmd;
1334         struct cuse_server_dev *pcsd;
1335         struct cuse_client *pcc;
1336         struct cuse_server *pcs;
1337         struct cuse_dev *pcd;
1338         int error;
1339         int n;
1340
1341         cuse_lock();
1342         pcsd = dev->si_drv1;
1343         if (pcsd != NULL) {
1344                 pcs = pcsd->server;
1345                 pcd = pcsd->user_dev;
1346                 pcs->refs++;
1347                 if (pcs->refs < 0) {
1348                         /* overflow */
1349                         pcs->refs--;
1350                         pcsd = NULL;
1351                 }
1352         } else {
1353                 pcs = NULL;
1354                 pcd = NULL;
1355         }
1356         cuse_unlock();
1357
1358         if (pcsd == NULL)
1359                 return (EINVAL);
1360
1361         pcc = malloc(sizeof(*pcc), M_CUSE, M_WAITOK | M_ZERO);
1362         if (pcc == NULL) {
1363                 /* drop reference on server */
1364                 cuse_server_free(pcs);
1365                 return (ENOMEM);
1366         }
1367         if (devfs_set_cdevpriv(pcc, &cuse_client_free)) {
1368                 printf("Cuse: Cannot set cdevpriv.\n");
1369                 /* drop reference on server */
1370                 cuse_server_free(pcs);
1371                 free(pcc, M_CUSE);
1372                 return (ENOMEM);
1373         }
1374         pcc->fflags = fflags;
1375         pcc->server_dev = pcsd;
1376         pcc->server = pcs;
1377
1378         for (n = 0; n != CUSE_CMD_MAX; n++) {
1379
1380                 pccmd = &pcc->cmds[n];
1381
1382                 pccmd->sub.dev = pcd;
1383                 pccmd->sub.command = n;
1384                 pccmd->client = pcc;
1385
1386                 sx_init(&pccmd->sx, "cuse-client-sx");
1387                 cv_init(&pccmd->cv, "cuse-client-cv");
1388         }
1389
1390         cuse_lock();
1391
1392         /* cuse_client_free() assumes that the client is listed somewhere! */
1393         /* always enqueue */
1394
1395         TAILQ_INSERT_TAIL(&pcs->hcli, pcc, entry);
1396
1397         /* check if server is closing */
1398         if ((pcs->is_closing != 0) || (dev->si_drv1 == NULL)) {
1399                 error = EINVAL;
1400         } else {
1401                 error = 0;
1402         }
1403         cuse_unlock();
1404
1405         if (error) {
1406                 devfs_clear_cdevpriv(); /* XXX bugfix */
1407                 return (error);
1408         }
1409         pccmd = &pcc->cmds[CUSE_CMD_OPEN];
1410
1411         cuse_cmd_lock(pccmd);
1412
1413         cuse_lock();
1414         cuse_client_send_command_locked(pccmd, 0, 0, pcc->fflags, 0);
1415
1416         error = cuse_client_receive_command_locked(pccmd, 0, 0);
1417         cuse_unlock();
1418
1419         if (error < 0) {
1420                 error = cuse_convert_error(error);
1421         } else {
1422                 error = 0;
1423         }
1424
1425         cuse_cmd_unlock(pccmd);
1426
1427         if (error)
1428                 devfs_clear_cdevpriv(); /* XXX bugfix */
1429
1430         return (error);
1431 }
1432
1433 static int
1434 cuse_client_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
1435 {
1436         struct cuse_client_command *pccmd;
1437         struct cuse_client *pcc;
1438         int error;
1439
1440         error = cuse_client_get(&pcc);
1441         if (error != 0)
1442                 return (0);
1443
1444         pccmd = &pcc->cmds[CUSE_CMD_CLOSE];
1445
1446         cuse_cmd_lock(pccmd);
1447
1448         cuse_lock();
1449         cuse_client_send_command_locked(pccmd, 0, 0, pcc->fflags, 0);
1450
1451         error = cuse_client_receive_command_locked(pccmd, 0, 0);
1452         cuse_unlock();
1453
1454         cuse_cmd_unlock(pccmd);
1455
1456         cuse_lock();
1457         cuse_client_is_closing(pcc);
1458         cuse_unlock();
1459
1460         return (0);
1461 }
1462
1463 static void
1464 cuse_client_kqfilter_poll(struct cdev *dev, struct cuse_client *pcc)
1465 {
1466         int temp;
1467
1468         cuse_lock();
1469         temp = (pcc->cflags & (CUSE_CLI_KNOTE_HAS_READ |
1470             CUSE_CLI_KNOTE_HAS_WRITE));
1471         pcc->cflags &= ~(CUSE_CLI_KNOTE_NEED_READ |
1472             CUSE_CLI_KNOTE_NEED_WRITE);
1473         cuse_unlock();
1474
1475         if (temp != 0) {
1476                 /* get the latest polling state from the server */
1477                 temp = cuse_client_poll(dev, POLLIN | POLLOUT, NULL);
1478
1479                 cuse_lock();
1480                 if (temp & (POLLIN | POLLOUT)) {
1481                         if (temp & POLLIN)
1482                                 pcc->cflags |= CUSE_CLI_KNOTE_NEED_READ;
1483                         if (temp & POLLOUT)
1484                                 pcc->cflags |= CUSE_CLI_KNOTE_NEED_WRITE;
1485
1486                         /* make sure the "knote" gets woken up */
1487                         cuse_server_wakeup_locked(pcc->server);
1488                 }
1489                 cuse_unlock();
1490         }
1491 }
1492
1493 static int
1494 cuse_client_read(struct cdev *dev, struct uio *uio, int ioflag)
1495 {
1496         struct cuse_client_command *pccmd;
1497         struct cuse_client *pcc;
1498         int error;
1499         int len;
1500
1501         error = cuse_client_get(&pcc);
1502         if (error != 0)
1503                 return (error);
1504
1505         pccmd = &pcc->cmds[CUSE_CMD_READ];
1506
1507         if (uio->uio_segflg != UIO_USERSPACE) {
1508                 return (EINVAL);
1509         }
1510         uio->uio_segflg = UIO_NOCOPY;
1511
1512         cuse_cmd_lock(pccmd);
1513
1514         while (uio->uio_resid != 0) {
1515
1516                 if (uio->uio_iov->iov_len > CUSE_LENGTH_MAX) {
1517                         error = ENOMEM;
1518                         break;
1519                 }
1520
1521                 len = uio->uio_iov->iov_len;
1522
1523                 cuse_lock();
1524                 cuse_client_send_command_locked(pccmd,
1525                     (unsigned long)uio->uio_iov->iov_base,
1526                     (unsigned long)(unsigned int)len, pcc->fflags, ioflag);
1527
1528                 error = cuse_client_receive_command_locked(pccmd, 0, 0);
1529                 cuse_unlock();
1530
1531                 if (error < 0) {
1532                         error = cuse_convert_error(error);
1533                         break;
1534                 } else if (error == len) {
1535                         error = uiomove(NULL, error, uio);
1536                         if (error)
1537                                 break;
1538                 } else {
1539                         error = uiomove(NULL, error, uio);
1540                         break;
1541                 }
1542         }
1543         cuse_cmd_unlock(pccmd);
1544
1545         uio->uio_segflg = UIO_USERSPACE;/* restore segment flag */
1546
1547         if (error == EWOULDBLOCK)
1548                 cuse_client_kqfilter_poll(dev, pcc);
1549
1550         return (error);
1551 }
1552
1553 static int
1554 cuse_client_write(struct cdev *dev, struct uio *uio, int ioflag)
1555 {
1556         struct cuse_client_command *pccmd;
1557         struct cuse_client *pcc;
1558         int error;
1559         int len;
1560
1561         error = cuse_client_get(&pcc);
1562         if (error != 0)
1563                 return (error);
1564
1565         pccmd = &pcc->cmds[CUSE_CMD_WRITE];
1566
1567         if (uio->uio_segflg != UIO_USERSPACE) {
1568                 return (EINVAL);
1569         }
1570         uio->uio_segflg = UIO_NOCOPY;
1571
1572         cuse_cmd_lock(pccmd);
1573
1574         while (uio->uio_resid != 0) {
1575
1576                 if (uio->uio_iov->iov_len > CUSE_LENGTH_MAX) {
1577                         error = ENOMEM;
1578                         break;
1579                 }
1580
1581                 len = uio->uio_iov->iov_len;
1582
1583                 cuse_lock();
1584                 cuse_client_send_command_locked(pccmd,
1585                     (unsigned long)uio->uio_iov->iov_base,
1586                     (unsigned long)(unsigned int)len, pcc->fflags, ioflag);
1587
1588                 error = cuse_client_receive_command_locked(pccmd, 0, 0);
1589                 cuse_unlock();
1590
1591                 if (error < 0) {
1592                         error = cuse_convert_error(error);
1593                         break;
1594                 } else if (error == len) {
1595                         error = uiomove(NULL, error, uio);
1596                         if (error)
1597                                 break;
1598                 } else {
1599                         error = uiomove(NULL, error, uio);
1600                         break;
1601                 }
1602         }
1603         cuse_cmd_unlock(pccmd);
1604
1605         uio->uio_segflg = UIO_USERSPACE;/* restore segment flag */
1606
1607         if (error == EWOULDBLOCK)
1608                 cuse_client_kqfilter_poll(dev, pcc);
1609
1610         return (error);
1611 }
1612
1613 int
1614 cuse_client_ioctl(struct cdev *dev, unsigned long cmd,
1615     caddr_t data, int fflag, struct thread *td)
1616 {
1617         struct cuse_client_command *pccmd;
1618         struct cuse_client *pcc;
1619         int error;
1620         int len;
1621
1622         error = cuse_client_get(&pcc);
1623         if (error != 0)
1624                 return (error);
1625
1626         len = IOCPARM_LEN(cmd);
1627         if (len > CUSE_BUFFER_MAX)
1628                 return (ENOMEM);
1629
1630         pccmd = &pcc->cmds[CUSE_CMD_IOCTL];
1631
1632         cuse_cmd_lock(pccmd);
1633
1634         if (cmd & IOC_IN)
1635                 memcpy(pcc->ioctl_buffer, data, len);
1636
1637         /*
1638          * When the ioctl-length is zero drivers can pass information
1639          * through the data pointer of the ioctl. Make sure this information
1640          * is forwarded to the driver.
1641          */
1642
1643         cuse_lock();
1644         cuse_client_send_command_locked(pccmd,
1645             (len == 0) ? *(long *)data : CUSE_BUF_MIN_PTR,
1646             (unsigned long)cmd, pcc->fflags,
1647             (fflag & O_NONBLOCK) ? IO_NDELAY : 0);
1648
1649         error = cuse_client_receive_command_locked(pccmd, data, len);
1650         cuse_unlock();
1651
1652         if (error < 0) {
1653                 error = cuse_convert_error(error);
1654         } else {
1655                 error = 0;
1656         }
1657
1658         if (cmd & IOC_OUT)
1659                 memcpy(data, pcc->ioctl_buffer, len);
1660
1661         cuse_cmd_unlock(pccmd);
1662
1663         if (error == EWOULDBLOCK)
1664                 cuse_client_kqfilter_poll(dev, pcc);
1665
1666         return (error);
1667 }
1668
1669 static int
1670 cuse_client_poll(struct cdev *dev, int events, struct thread *td)
1671 {
1672         struct cuse_client_command *pccmd;
1673         struct cuse_client *pcc;
1674         unsigned long temp;
1675         int error;
1676         int revents;
1677
1678         error = cuse_client_get(&pcc);
1679         if (error != 0)
1680                 return (POLLNVAL);
1681
1682         temp = 0;
1683
1684         if (events & (POLLPRI | POLLIN | POLLRDNORM))
1685                 temp |= CUSE_POLL_READ;
1686
1687         if (events & (POLLOUT | POLLWRNORM))
1688                 temp |= CUSE_POLL_WRITE;
1689
1690         if (events & POLLHUP)
1691                 temp |= CUSE_POLL_ERROR;
1692
1693         pccmd = &pcc->cmds[CUSE_CMD_POLL];
1694
1695         cuse_cmd_lock(pccmd);
1696
1697         /* Need to selrecord() first to not loose any events. */
1698         if (temp != 0 && td != NULL)
1699                 selrecord(td, &pcc->server->selinfo);
1700
1701         cuse_lock();
1702         cuse_client_send_command_locked(pccmd,
1703             0, temp, pcc->fflags, IO_NDELAY);
1704
1705         error = cuse_client_receive_command_locked(pccmd, 0, 0);
1706         cuse_unlock();
1707
1708         if (error < 0) {
1709                 revents = POLLNVAL;
1710         } else {
1711                 revents = 0;
1712                 if (error & CUSE_POLL_READ)
1713                         revents |= (events & (POLLPRI | POLLIN | POLLRDNORM));
1714                 if (error & CUSE_POLL_WRITE)
1715                         revents |= (events & (POLLOUT | POLLWRNORM));
1716                 if (error & CUSE_POLL_ERROR)
1717                         revents |= (events & POLLHUP);
1718         }
1719
1720         cuse_cmd_unlock(pccmd);
1721
1722         return (revents);
1723 }
1724
1725 static int
1726 cuse_client_mmap(struct cdev *dev, vm_ooffset_t offset, vm_paddr_t *paddr, int nprot, vm_memattr_t *memattr)
1727 {
1728         uint32_t page_nr = offset / PAGE_SIZE;
1729         uint32_t alloc_nr = page_nr / CUSE_ALLOC_PAGES_MAX;
1730         struct cuse_memory *mem;
1731         struct cuse_server *pcs;
1732         struct cuse_client *pcc;
1733         uint8_t *ptr;
1734         int error;
1735
1736         if (alloc_nr >= CUSE_ALLOC_UNIT_MAX)
1737                 return (ENOMEM);
1738
1739         error = cuse_client_get(&pcc);
1740         if (error != 0)
1741                 pcs = NULL;
1742         else
1743                 pcs = pcc->server;
1744
1745         cuse_lock();
1746         mem = &cuse_mem[alloc_nr];
1747
1748         /* try to enforce slight ownership */
1749         if ((pcs != NULL) && (mem->owner != pcs)) {
1750                 cuse_unlock();
1751                 return (EINVAL);
1752         }
1753         if (mem->virtaddr == NULL) {
1754                 cuse_unlock();
1755                 return (ENOMEM);
1756         }
1757         if (mem->virtaddr == NBUSY) {
1758                 cuse_unlock();
1759                 return (ENOMEM);
1760         }
1761         page_nr %= CUSE_ALLOC_PAGES_MAX;
1762
1763         if (page_nr >= mem->page_count) {
1764                 cuse_unlock();
1765                 return (ENXIO);
1766         }
1767         ptr = mem->virtaddr + (page_nr * PAGE_SIZE);
1768         cuse_unlock();
1769
1770         *paddr = vtophys(ptr);
1771
1772         return (0);
1773 }
1774
1775 static void
1776 cuse_client_kqfilter_read_detach(struct knote *kn)
1777 {
1778         struct cuse_client *pcc;
1779
1780         cuse_lock();
1781         pcc = kn->kn_hook;
1782         knlist_remove(&pcc->server->selinfo.si_note, kn, 1);
1783         cuse_unlock();
1784 }
1785
1786 static void
1787 cuse_client_kqfilter_write_detach(struct knote *kn)
1788 {
1789         struct cuse_client *pcc;
1790
1791         cuse_lock();
1792         pcc = kn->kn_hook;
1793         knlist_remove(&pcc->server->selinfo.si_note, kn, 1);
1794         cuse_unlock();
1795 }
1796
1797 static int
1798 cuse_client_kqfilter_read_event(struct knote *kn, long hint)
1799 {
1800         struct cuse_client *pcc;
1801
1802         mtx_assert(&cuse_mtx, MA_OWNED);
1803
1804         pcc = kn->kn_hook;
1805         return ((pcc->cflags & CUSE_CLI_KNOTE_NEED_READ) ? 1 : 0);
1806 }
1807
1808 static int
1809 cuse_client_kqfilter_write_event(struct knote *kn, long hint)
1810 {
1811         struct cuse_client *pcc;
1812
1813         mtx_assert(&cuse_mtx, MA_OWNED);
1814
1815         pcc = kn->kn_hook;
1816         return ((pcc->cflags & CUSE_CLI_KNOTE_NEED_WRITE) ? 1 : 0);
1817 }
1818
1819 static int
1820 cuse_client_kqfilter(struct cdev *dev, struct knote *kn)
1821 {
1822         struct cuse_client *pcc;
1823         struct cuse_server *pcs;
1824         int error;
1825
1826         error = cuse_client_get(&pcc);
1827         if (error != 0)
1828                 return (error);
1829
1830         cuse_lock();
1831         pcs = pcc->server;
1832         switch (kn->kn_filter) {
1833         case EVFILT_READ:
1834                 pcc->cflags |= CUSE_CLI_KNOTE_HAS_READ;
1835                 kn->kn_hook = pcc;
1836                 kn->kn_fop = &cuse_client_kqfilter_read_ops;
1837                 knlist_add(&pcs->selinfo.si_note, kn, 1);
1838                 break;
1839         case EVFILT_WRITE:
1840                 pcc->cflags |= CUSE_CLI_KNOTE_HAS_WRITE;
1841                 kn->kn_hook = pcc;
1842                 kn->kn_fop = &cuse_client_kqfilter_write_ops;
1843                 knlist_add(&pcs->selinfo.si_note, kn, 1);
1844                 break;
1845         default:
1846                 error = EINVAL;
1847                 break;
1848         }
1849         cuse_unlock();
1850
1851         if (error == 0)
1852                 cuse_client_kqfilter_poll(dev, pcc);
1853         return (error);
1854 }