sys/compat/linuxkpi/common/src/linux_compat.c

   1 /*-
   2  * Copyright (c) 2010 Isilon Systems, Inc.
   3  * Copyright (c) 2010 iX Systems, Inc.
   4  * Copyright (c) 2010 Panasas, Inc.
   5  * Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice unmodified, this list of conditions, and the following
  13  *    disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  23  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  27  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/malloc.h>
  36 #include <sys/kernel.h>
  37 #include <sys/sysctl.h>
  38 #include <sys/proc.h>
  39 #include <sys/sglist.h>
  40 #include <sys/sleepqueue.h>
  41 #include <sys/lock.h>
  42 #include <sys/mutex.h>
  43 #include <sys/bus.h>
  44 #include <sys/fcntl.h>
  45 #include <sys/file.h>
  46 #include <sys/filio.h>
  47 #include <sys/rwlock.h>
  48
  49 #include <vm/vm.h>
  50 #include <vm/pmap.h>
  51 #include <vm/vm_object.h>
  52 #include <vm/vm_page.h>
  53 #include <vm/vm_pager.h>
  54
  55 #include <machine/stdarg.h>
  56
  57 #if defined(__i386__) || defined(__amd64__)
  58 #include <machine/md_var.h>
  59 #endif
  60
  61 #include <linux/kobject.h>
  62 #include <linux/device.h>
  63 #include <linux/slab.h>
  64 #include <linux/module.h>
  65 #include <linux/moduleparam.h>
  66 #include <linux/cdev.h>
  67 #include <linux/file.h>
  68 #include <linux/sysfs.h>
  69 #include <linux/mm.h>
  70 #include <linux/io.h>
  71 #include <linux/vmalloc.h>
  72 #include <linux/netdevice.h>
  73 #include <linux/timer.h>
  74 #include <linux/interrupt.h>
  75 #include <linux/uaccess.h>
  76 #include <linux/list.h>
  77 #include <linux/kthread.h>
  78 #include <linux/kernel.h>
  79 #include <linux/compat.h>
  80 #include <linux/poll.h>
  81 #include <linux/smp.h>
  82
  83 #if defined(__i386__) || defined(__amd64__)
  84 #include <asm/smp.h>
  85 #endif
  86
  87 SYSCTL_NODE(_compat, OID_AUTO, linuxkpi, CTLFLAG_RW, 0, "LinuxKPI parameters");
  88
  89 MALLOC_DEFINE(M_KMALLOC, "linux", "Linux kmalloc compat");
  90
  91 #include <linux/rbtree.h>
  92 /* Undo Linux compat changes. */
  93 #undef RB_ROOT
  94 #undef file
  95 #undef cdev
  96 #define RB_ROOT(head)   (head)->rbh_root
  97
  98 static struct vm_area_struct *linux_cdev_handle_find(void *handle);
  99
 100 struct kobject linux_class_root;
 101 struct device linux_root_device;
 102 struct class linux_class_misc;
 103 struct list_head pci_drivers;
 104 struct list_head pci_devices;
 105 spinlock_t pci_lock;
 106
 107 unsigned long linux_timer_hz_mask;
 108
 109 int
 110 panic_cmp(struct rb_node *one, struct rb_node *two)
 111 {
 112         panic("no cmp");
 113 }
 114
 115 RB_GENERATE(linux_root, rb_node, __entry, panic_cmp);
 116
 117 int
 118 kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list args)
 119 {
 120         va_list tmp_va;
 121         int len;
 122         char *old;
 123         char *name;
 124         char dummy;
 125
 126         old = kobj->name;
 127
 128         if (old && fmt == NULL)
 129                 return (0);
 130
 131         /* compute length of string */
 132         va_copy(tmp_va, args);
 133         len = vsnprintf(&dummy, 0, fmt, tmp_va);
 134         va_end(tmp_va);
 135
 136         /* account for zero termination */
 137         len++;
 138
 139         /* check for error */
 140         if (len < 1)
 141                 return (-EINVAL);
 142
 143         /* allocate memory for string */
 144         name = kzalloc(len, GFP_KERNEL);
 145         if (name == NULL)
 146                 return (-ENOMEM);
 147         vsnprintf(name, len, fmt, args);
 148         kobj->name = name;
 149
 150         /* free old string */
 151         kfree(old);
 152
 153         /* filter new string */
 154         for (; *name != '\0'; name++)
 155                 if (*name == '/')
 156                         *name = '!';
 157         return (0);
 158 }
 159
 160 int
 161 kobject_set_name(struct kobject *kobj, const char *fmt, ...)
 162 {
 163         va_list args;
 164         int error;
 165
 166         va_start(args, fmt);
 167         error = kobject_set_name_vargs(kobj, fmt, args);
 168         va_end(args);
 169
 170         return (error);
 171 }
 172
 173 static int
 174 kobject_add_complete(struct kobject *kobj, struct kobject *parent)
 175 {
 176         const struct kobj_type *t;
 177         int error;
 178
 179         kobj->parent = parent;
 180         error = sysfs_create_dir(kobj);
 181         if (error == 0 && kobj->ktype && kobj->ktype->default_attrs) {
 182                 struct attribute **attr;
 183                 t = kobj->ktype;
 184
 185                 for (attr = t->default_attrs; *attr != NULL; attr++) {
 186                         error = sysfs_create_file(kobj, *attr);
 187                         if (error)
 188                                 break;
 189                 }
 190                 if (error)
 191                         sysfs_remove_dir(kobj);
 192
 193         }
 194         return (error);
 195 }
 196
 197 int
 198 kobject_add(struct kobject *kobj, struct kobject *parent, const char *fmt, ...)
 199 {
 200         va_list args;
 201         int error;
 202
 203         va_start(args, fmt);
 204         error = kobject_set_name_vargs(kobj, fmt, args);
 205         va_end(args);
 206         if (error)
 207                 return (error);
 208
 209         return kobject_add_complete(kobj, parent);
 210 }
 211
 212 void
 213 linux_kobject_release(struct kref *kref)
 214 {
 215         struct kobject *kobj;
 216         char *name;
 217
 218         kobj = container_of(kref, struct kobject, kref);
 219         sysfs_remove_dir(kobj);
 220         name = kobj->name;
 221         if (kobj->ktype && kobj->ktype->release)
 222                 kobj->ktype->release(kobj);
 223         kfree(name);
 224 }
 225
 226 static void
 227 linux_kobject_kfree(struct kobject *kobj)
 228 {
 229         kfree(kobj);
 230 }
 231
 232 static void
 233 linux_kobject_kfree_name(struct kobject *kobj)
 234 {
 235         if (kobj) {
 236                 kfree(kobj->name);
 237         }
 238 }
 239
 240 const struct kobj_type linux_kfree_type = {
 241         .release = linux_kobject_kfree
 242 };
 243
 244 static void
 245 linux_device_release(struct device *dev)
 246 {
 247         pr_debug("linux_device_release: %s\n", dev_name(dev));
 248         kfree(dev);
 249 }
 250
 251 static ssize_t
 252 linux_class_show(struct kobject *kobj, struct attribute *attr, char *buf)
 253 {
 254         struct class_attribute *dattr;
 255         ssize_t error;
 256
 257         dattr = container_of(attr, struct class_attribute, attr);
 258         error = -EIO;
 259         if (dattr->show)
 260                 error = dattr->show(container_of(kobj, struct class, kobj),
 261                     dattr, buf);
 262         return (error);
 263 }
 264
 265 static ssize_t
 266 linux_class_store(struct kobject *kobj, struct attribute *attr, const char *buf,
 267     size_t count)
 268 {
 269         struct class_attribute *dattr;
 270         ssize_t error;
 271
 272         dattr = container_of(attr, struct class_attribute, attr);
 273         error = -EIO;
 274         if (dattr->store)
 275                 error = dattr->store(container_of(kobj, struct class, kobj),
 276                     dattr, buf, count);
 277         return (error);
 278 }
 279
 280 static void
 281 linux_class_release(struct kobject *kobj)
 282 {
 283         struct class *class;
 284
 285         class = container_of(kobj, struct class, kobj);
 286         if (class->class_release)
 287                 class->class_release(class);
 288 }
 289
 290 static const struct sysfs_ops linux_class_sysfs = {
 291         .show  = linux_class_show,
 292         .store = linux_class_store,
 293 };
 294
 295 const struct kobj_type linux_class_ktype = {
 296         .release = linux_class_release,
 297         .sysfs_ops = &linux_class_sysfs
 298 };
 299
 300 static void
 301 linux_dev_release(struct kobject *kobj)
 302 {
 303         struct device *dev;
 304
 305         dev = container_of(kobj, struct device, kobj);
 306         /* This is the precedence defined by linux. */
 307         if (dev->release)
 308                 dev->release(dev);
 309         else if (dev->class && dev->class->dev_release)
 310                 dev->class->dev_release(dev);
 311 }
 312
 313 static ssize_t
 314 linux_dev_show(struct kobject *kobj, struct attribute *attr, char *buf)
 315 {
 316         struct device_attribute *dattr;
 317         ssize_t error;
 318
 319         dattr = container_of(attr, struct device_attribute, attr);
 320         error = -EIO;
 321         if (dattr->show)
 322                 error = dattr->show(container_of(kobj, struct device, kobj),
 323                     dattr, buf);
 324         return (error);
 325 }
 326
 327 static ssize_t
 328 linux_dev_store(struct kobject *kobj, struct attribute *attr, const char *buf,
 329     size_t count)
 330 {
 331         struct device_attribute *dattr;
 332         ssize_t error;
 333
 334         dattr = container_of(attr, struct device_attribute, attr);
 335         error = -EIO;
 336         if (dattr->store)
 337                 error = dattr->store(container_of(kobj, struct device, kobj),
 338                     dattr, buf, count);
 339         return (error);
 340 }
 341
 342 static const struct sysfs_ops linux_dev_sysfs = {
 343         .show  = linux_dev_show,
 344         .store = linux_dev_store,
 345 };
 346
 347 const struct kobj_type linux_dev_ktype = {
 348         .release = linux_dev_release,
 349         .sysfs_ops = &linux_dev_sysfs
 350 };
 351
 352 struct device *
 353 device_create(struct class *class, struct device *parent, dev_t devt,
 354     void *drvdata, const char *fmt, ...)
 355 {
 356         struct device *dev;
 357         va_list args;
 358
 359         dev = kzalloc(sizeof(*dev), M_WAITOK);
 360         dev->parent = parent;
 361         dev->class = class;
 362         dev->devt = devt;
 363         dev->driver_data = drvdata;
 364         dev->release = linux_device_release;
 365         va_start(args, fmt);
 366         kobject_set_name_vargs(&dev->kobj, fmt, args);
 367         va_end(args);
 368         device_register(dev);
 369
 370         return (dev);
 371 }
 372
 373 int
 374 kobject_init_and_add(struct kobject *kobj, const struct kobj_type *ktype,
 375     struct kobject *parent, const char *fmt, ...)
 376 {
 377         va_list args;
 378         int error;
 379
 380         kobject_init(kobj, ktype);
 381         kobj->ktype = ktype;
 382         kobj->parent = parent;
 383         kobj->name = NULL;
 384
 385         va_start(args, fmt);
 386         error = kobject_set_name_vargs(kobj, fmt, args);
 387         va_end(args);
 388         if (error)
 389                 return (error);
 390         return kobject_add_complete(kobj, parent);
 391 }
 392
 393 static void
 394 linux_file_dtor(void *cdp)
 395 {
 396         struct linux_file *filp;
 397
 398         linux_set_current(curthread);
 399         filp = cdp;
 400         filp->f_op->release(filp->f_vnode, filp);
 401         vdrop(filp->f_vnode);
 402         kfree(filp);
 403 }
 404
 405 static void
 406 linux_kq_lock(void *arg)
 407 {
 408         spinlock_t *s = arg;
 409
 410         spin_lock(s);
 411 }
 412 static void
 413 linux_kq_unlock(void *arg)
 414 {
 415         spinlock_t *s = arg;
 416
 417         spin_unlock(s);
 418 }
 419
 420 static void
 421 linux_kq_lock_owned(void *arg)
 422 {
 423 #ifdef INVARIANTS
 424         spinlock_t *s = arg;
 425
 426         mtx_assert(&s->m, MA_OWNED);
 427 #endif
 428 }
 429
 430 static void
 431 linux_kq_lock_unowned(void *arg)
 432 {
 433 #ifdef INVARIANTS
 434         spinlock_t *s = arg;
 435
 436         mtx_assert(&s->m, MA_NOTOWNED);
 437 #endif
 438 }
 439
 440 static void
 441 linux_dev_kqfilter_poll(struct linux_file *, int);
 442
 443 struct linux_file *
 444 linux_file_alloc(void)
 445 {
 446         struct linux_file *filp;
 447
 448         filp = kzalloc(sizeof(*filp), GFP_KERNEL);
 449
 450         /* set initial refcount */
 451         filp->f_count = 1;
 452
 453         /* setup fields needed by kqueue support */
 454         spin_lock_init(&filp->f_kqlock);
 455         knlist_init(&filp->f_selinfo.si_note, &filp->f_kqlock,
 456             linux_kq_lock, linux_kq_unlock,
 457             linux_kq_lock_owned, linux_kq_lock_unowned);
 458
 459         return (filp);
 460 }
 461
 462 void
 463 linux_file_free(struct linux_file *filp)
 464 {
 465         if (filp->_file == NULL) {
 466                 if (filp->f_shmem != NULL)
 467                         vm_object_deallocate(filp->f_shmem);
 468                 kfree(filp);
 469         } else {
 470                 /*
 471                  * The close method of the character device or file
 472                  * will free the linux_file structure:
 473                  */
 474                 _fdrop(filp->_file, curthread);
 475         }
 476 }
 477
 478 static int
 479 linux_cdev_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot,
 480     vm_page_t *mres)
 481 {
 482         struct vm_area_struct *vmap;
 483
 484         vmap = linux_cdev_handle_find(vm_obj->handle);
 485
 486         MPASS(vmap != NULL);
 487         MPASS(vmap->vm_private_data == vm_obj->handle);
 488
 489         if (likely(vmap->vm_ops != NULL && offset < vmap->vm_len)) {
 490                 vm_paddr_t paddr = IDX_TO_OFF(vmap->vm_pfn) + offset;
 491                 vm_page_t page;
 492
 493                 if (((*mres)->flags & PG_FICTITIOUS) != 0) {
 494                         /*
 495                          * If the passed in result page is a fake
 496                          * page, update it with the new physical
 497                          * address.
 498                          */
 499                         page = *mres;
 500                         vm_page_updatefake(page, paddr, vm_obj->memattr);
 501                 } else {
 502                         /*
 503                          * Replace the passed in "mres" page with our
 504                          * own fake page and free up the all of the
 505                          * original pages.
 506                          */
 507                         VM_OBJECT_WUNLOCK(vm_obj);
 508                         page = vm_page_getfake(paddr, vm_obj->memattr);
 509                         VM_OBJECT_WLOCK(vm_obj);
 510
 511                         vm_page_replace_checked(page, vm_obj,
 512                             (*mres)->pindex, *mres);
 513
 514                         vm_page_lock(*mres);
 515                         vm_page_free(*mres);
 516                         vm_page_unlock(*mres);
 517                         *mres = page;
 518                 }
 519                 page->valid = VM_PAGE_BITS_ALL;
 520                 return (VM_PAGER_OK);
 521         }
 522         return (VM_PAGER_FAIL);
 523 }
 524
 525 static int
 526 linux_cdev_pager_populate(vm_object_t vm_obj, vm_pindex_t pidx, int fault_type,
 527     vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last)
 528 {
 529         struct vm_area_struct *vmap;
 530         int err;
 531
 532         linux_set_current(curthread);
 533
 534         /* get VM area structure */
 535         vmap = linux_cdev_handle_find(vm_obj->handle);
 536         MPASS(vmap != NULL);
 537         MPASS(vmap->vm_private_data == vm_obj->handle);
 538
 539         VM_OBJECT_WUNLOCK(vm_obj);
 540
 541         down_write(&vmap->vm_mm->mmap_sem);
 542         if (unlikely(vmap->vm_ops == NULL)) {
 543                 err = VM_FAULT_SIGBUS;
 544         } else {
 545                 struct vm_fault vmf;
 546
 547                 /* fill out VM fault structure */
 548                 vmf.virtual_address = (void *)((uintptr_t)pidx << PAGE_SHIFT);
 549                 vmf.flags = (fault_type & VM_PROT_WRITE) ? FAULT_FLAG_WRITE : 0;
 550                 vmf.pgoff = 0;
 551                 vmf.page = NULL;
 552
 553                 vmap->vm_pfn_count = 0;
 554                 vmap->vm_pfn_pcount = &vmap->vm_pfn_count;
 555                 vmap->vm_obj = vm_obj;
 556
 557                 err = vmap->vm_ops->fault(vmap, &vmf);
 558
 559                 while (vmap->vm_pfn_count == 0 && err == VM_FAULT_NOPAGE) {
 560                         kern_yield(PRI_USER);
 561                         err = vmap->vm_ops->fault(vmap, &vmf);
 562                 }
 563         }
 564
 565         /* translate return code */
 566         switch (err) {
 567         case VM_FAULT_OOM:
 568                 err = VM_PAGER_AGAIN;
 569                 break;
 570         case VM_FAULT_SIGBUS:
 571                 err = VM_PAGER_BAD;
 572                 break;
 573         case VM_FAULT_NOPAGE:
 574                 /*
 575                  * By contract the fault handler will return having
 576                  * busied all the pages itself. If pidx is already
 577                  * found in the object, it will simply xbusy the first
 578                  * page and return with vm_pfn_count set to 1.
 579                  */
 580                 *first = vmap->vm_pfn_first;
 581                 *last = *first + vmap->vm_pfn_count - 1;
 582                 err = VM_PAGER_OK;
 583                 break;
 584         default:
 585                 err = VM_PAGER_ERROR;
 586                 break;
 587         }
 588         up_write(&vmap->vm_mm->mmap_sem);
 589         VM_OBJECT_WLOCK(vm_obj);
 590         return (err);
 591 }
 592
 593 static struct rwlock linux_vma_lock;
 594 static TAILQ_HEAD(, vm_area_struct) linux_vma_head =
 595     TAILQ_HEAD_INITIALIZER(linux_vma_head);
 596
 597 static void
 598 linux_cdev_handle_free(struct vm_area_struct *vmap)
 599 {
 600         /* Drop reference on vm_file */
 601         if (vmap->vm_file != NULL)
 602                 fput(vmap->vm_file);
 603
 604         /* Drop reference on mm_struct */
 605         mmput(vmap->vm_mm);
 606
 607         kfree(vmap);
 608 }
 609
 610 static struct vm_area_struct *
 611 linux_cdev_handle_insert(void *handle, struct vm_area_struct *vmap)
 612 {
 613         struct vm_area_struct *ptr;
 614
 615         rw_wlock(&linux_vma_lock);
 616         TAILQ_FOREACH(ptr, &linux_vma_head, vm_entry) {
 617                 if (ptr->vm_private_data == handle) {
 618                         rw_wunlock(&linux_vma_lock);
 619                         linux_cdev_handle_free(vmap);
 620                         return (NULL);
 621                 }
 622         }
 623         TAILQ_INSERT_TAIL(&linux_vma_head, vmap, vm_entry);
 624         rw_wunlock(&linux_vma_lock);
 625         return (vmap);
 626 }
 627
 628 static void
 629 linux_cdev_handle_remove(struct vm_area_struct *vmap)
 630 {
 631         rw_wlock(&linux_vma_lock);
 632         TAILQ_REMOVE(&linux_vma_head, vmap, vm_entry);
 633         rw_wunlock(&linux_vma_lock);
 634 }
 635
 636 static struct vm_area_struct *
 637 linux_cdev_handle_find(void *handle)
 638 {
 639         struct vm_area_struct *vmap;
 640
 641         rw_rlock(&linux_vma_lock);
 642         TAILQ_FOREACH(vmap, &linux_vma_head, vm_entry) {
 643                 if (vmap->vm_private_data == handle)
 644                         break;
 645         }
 646         rw_runlock(&linux_vma_lock);
 647         return (vmap);
 648 }
 649
 650 static int
 651 linux_cdev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
 652                       vm_ooffset_t foff, struct ucred *cred, u_short *color)
 653 {
 654
 655         MPASS(linux_cdev_handle_find(handle) != NULL);
 656         *color = 0;
 657         return (0);
 658 }
 659
 660 static void
 661 linux_cdev_pager_dtor(void *handle)
 662 {
 663         const struct vm_operations_struct *vm_ops;
 664         struct vm_area_struct *vmap;
 665
 666         vmap = linux_cdev_handle_find(handle);
 667         MPASS(vmap != NULL);
 668
 669         /*
 670          * Remove handle before calling close operation to prevent
 671          * other threads from reusing the handle pointer.
 672          */
 673         linux_cdev_handle_remove(vmap);
 674
 675         down_write(&vmap->vm_mm->mmap_sem);
 676         vm_ops = vmap->vm_ops;
 677         if (likely(vm_ops != NULL))
 678                 vm_ops->close(vmap);
 679         up_write(&vmap->vm_mm->mmap_sem);
 680
 681         linux_cdev_handle_free(vmap);
 682 }
 683
 684 static struct cdev_pager_ops linux_cdev_pager_ops[2] = {
 685   {
 686         /* OBJT_MGTDEVICE */
 687         .cdev_pg_populate       = linux_cdev_pager_populate,
 688         .cdev_pg_ctor   = linux_cdev_pager_ctor,
 689         .cdev_pg_dtor   = linux_cdev_pager_dtor
 690   },
 691   {
 692         /* OBJT_DEVICE */
 693         .cdev_pg_fault  = linux_cdev_pager_fault,
 694         .cdev_pg_ctor   = linux_cdev_pager_ctor,
 695         .cdev_pg_dtor   = linux_cdev_pager_dtor
 696   },
 697 };
 698
 699 static int
 700 linux_dev_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
 701 {
 702         struct linux_cdev *ldev;
 703         struct linux_file *filp;
 704         struct file *file;
 705         int error;
 706
 707         file = td->td_fpop;
 708         ldev = dev->si_drv1;
 709         if (ldev == NULL)
 710                 return (ENODEV);
 711
 712         filp = linux_file_alloc();
 713         filp->f_dentry = &filp->f_dentry_store;
 714         filp->f_op = ldev->ops;
 715         filp->f_flags = file->f_flag;
 716         vhold(file->f_vnode);
 717         filp->f_vnode = file->f_vnode;
 718         filp->_file = file;
 719
 720         linux_set_current(td);
 721
 722         if (filp->f_op->open) {
 723                 error = -filp->f_op->open(file->f_vnode, filp);
 724                 if (error) {
 725                         vdrop(filp->f_vnode);
 726                         kfree(filp);
 727                         goto done;
 728                 }
 729         }
 730         error = devfs_set_cdevpriv(filp, linux_file_dtor);
 731         if (error) {
 732                 filp->f_op->release(file->f_vnode, filp);
 733                 vdrop(filp->f_vnode);
 734                 kfree(filp);
 735         }
 736 done:
 737         return (error);
 738 }
 739
 740 static int
 741 linux_dev_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
 742 {
 743         struct linux_file *filp;
 744         struct file *file;
 745         int error;
 746
 747         file = td->td_fpop;
 748         if (dev->si_drv1 == NULL)
 749                 return (0);
 750         if ((error = devfs_get_cdevpriv((void **)&filp)) != 0)
 751                 return (error);
 752         filp->f_flags = file->f_flag;
 753         devfs_clear_cdevpriv();
 754
 755         return (0);
 756 }
 757
 758 #define LINUX_IOCTL_MIN_PTR 0x10000UL
 759 #define LINUX_IOCTL_MAX_PTR (LINUX_IOCTL_MIN_PTR + IOCPARM_MAX)
 760
 761 static inline int
 762 linux_remap_address(void **uaddr, size_t len)
 763 {
 764         uintptr_t uaddr_val = (uintptr_t)(*uaddr);
 765
 766         if (unlikely(uaddr_val >= LINUX_IOCTL_MIN_PTR &&
 767             uaddr_val < LINUX_IOCTL_MAX_PTR)) {
 768                 struct task_struct *pts = current;
 769                 if (pts == NULL) {
 770                         *uaddr = NULL;
 771                         return (1);
 772                 }
 773
 774                 /* compute data offset */
 775                 uaddr_val -= LINUX_IOCTL_MIN_PTR;
 776
 777                 /* check that length is within bounds */
 778                 if ((len > IOCPARM_MAX) ||
 779                     (uaddr_val + len) > pts->bsd_ioctl_len) {
 780                         *uaddr = NULL;
 781                         return (1);
 782                 }
 783
 784                 /* re-add kernel buffer address */
 785                 uaddr_val += (uintptr_t)pts->bsd_ioctl_data;
 786
 787                 /* update address location */
 788                 *uaddr = (void *)uaddr_val;
 789                 return (1);
 790         }
 791         return (0);
 792 }
 793
 794 int
 795 linux_copyin(const void *uaddr, void *kaddr, size_t len)
 796 {
 797         if (linux_remap_address(__DECONST(void **, &uaddr), len)) {
 798                 if (uaddr == NULL)
 799                         return (-EFAULT);
 800                 memcpy(kaddr, uaddr, len);
 801                 return (0);
 802         }
 803         return (-copyin(uaddr, kaddr, len));
 804 }
 805
 806 int
 807 linux_copyout(const void *kaddr, void *uaddr, size_t len)
 808 {
 809         if (linux_remap_address(&uaddr, len)) {
 810                 if (uaddr == NULL)
 811                         return (-EFAULT);
 812                 memcpy(uaddr, kaddr, len);
 813                 return (0);
 814         }
 815         return (-copyout(kaddr, uaddr, len));
 816 }
 817
 818 size_t
 819 linux_clear_user(void *_uaddr, size_t _len)
 820 {
 821         uint8_t *uaddr = _uaddr;
 822         size_t len = _len;
 823
 824         /* make sure uaddr is aligned before going into the fast loop */
 825         while (((uintptr_t)uaddr & 7) != 0 && len > 7) {
 826                 if (subyte(uaddr, 0))
 827                         return (_len);
 828                 uaddr++;
 829                 len--;
 830         }
 831
 832         /* zero 8 bytes at a time */
 833         while (len > 7) {
 834 #ifdef __LP64__
 835                 if (suword64(uaddr, 0))
 836                         return (_len);
 837 #else
 838                 if (suword32(uaddr, 0))
 839                         return (_len);
 840                 if (suword32(uaddr + 4, 0))
 841                         return (_len);
 842 #endif
 843                 uaddr += 8;
 844                 len -= 8;
 845         }
 846
 847         /* zero fill end, if any */
 848         while (len > 0) {
 849                 if (subyte(uaddr, 0))
 850                         return (_len);
 851                 uaddr++;
 852                 len--;
 853         }
 854         return (0);
 855 }
 856
 857 int
 858 linux_access_ok(int rw, const void *uaddr, size_t len)
 859 {
 860         uintptr_t saddr;
 861         uintptr_t eaddr;
 862
 863         /* get start and end address */
 864         saddr = (uintptr_t)uaddr;
 865         eaddr = (uintptr_t)uaddr + len;
 866
 867         /* verify addresses are valid for userspace */
 868         return ((saddr == eaddr) ||
 869             (eaddr > saddr && eaddr <= VM_MAXUSER_ADDRESS));
 870 }
 871
 872 static int
 873 linux_dev_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
 874     struct thread *td)
 875 {
 876         struct linux_file *filp;
 877         struct file *file;
 878         unsigned size;
 879         int error;
 880
 881         file = td->td_fpop;
 882         if (dev->si_drv1 == NULL)
 883                 return (ENXIO);
 884         if ((error = devfs_get_cdevpriv((void **)&filp)) != 0)
 885                 return (error);
 886         filp->f_flags = file->f_flag;
 887
 888         /* the LinuxKPI supports blocking and non-blocking I/O */
 889         if (cmd == FIONBIO || cmd == FIOASYNC)
 890                 return (0);
 891
 892         linux_set_current(td);
 893         size = IOCPARM_LEN(cmd);
 894         /* refer to logic in sys_ioctl() */
 895         if (size > 0) {
 896                 /*
 897                  * Setup hint for linux_copyin() and linux_copyout().
 898                  *
 899                  * Background: Linux code expects a user-space address
 900                  * while FreeBSD supplies a kernel-space address.
 901                  */
 902                 current->bsd_ioctl_data = data;
 903                 current->bsd_ioctl_len = size;
 904                 data = (void *)LINUX_IOCTL_MIN_PTR;
 905         } else {
 906                 /* fetch user-space pointer */
 907                 data = *(void **)data;
 908         }
 909         if (filp->f_op->unlocked_ioctl)
 910                 error = -filp->f_op->unlocked_ioctl(filp, cmd, (u_long)data);
 911         else
 912                 error = ENOTTY;
 913         if (size > 0) {
 914                 current->bsd_ioctl_data = NULL;
 915                 current->bsd_ioctl_len = 0;
 916         }
 917
 918         if (error == EWOULDBLOCK) {
 919                 /* update kqfilter status, if any */
 920                 linux_dev_kqfilter_poll(filp,
 921                     LINUX_KQ_FLAG_HAS_READ | LINUX_KQ_FLAG_HAS_WRITE);
 922         } else if (error == ERESTARTSYS)
 923                 error = ERESTART;
 924         return (error);
 925 }
 926
 927 static int
 928 linux_dev_read(struct cdev *dev, struct uio *uio, int ioflag)
 929 {
 930         struct linux_file *filp;
 931         struct thread *td;
 932         struct file *file;
 933         ssize_t bytes;
 934         int error;
 935
 936         td = curthread;
 937         file = td->td_fpop;
 938         if (dev->si_drv1 == NULL)
 939                 return (ENXIO);
 940         if ((error = devfs_get_cdevpriv((void **)&filp)) != 0)
 941                 return (error);
 942         filp->f_flags = file->f_flag;
 943         /* XXX no support for I/O vectors currently */
 944         if (uio->uio_iovcnt != 1)
 945                 return (EOPNOTSUPP);
 946         linux_set_current(td);
 947         if (filp->f_op->read) {
 948                 bytes = filp->f_op->read(filp, uio->uio_iov->iov_base,
 949                     uio->uio_iov->iov_len, &uio->uio_offset);
 950                 if (bytes >= 0) {
 951                         uio->uio_iov->iov_base =
 952                             ((uint8_t *)uio->uio_iov->iov_base) + bytes;
 953                         uio->uio_iov->iov_len -= bytes;
 954                         uio->uio_resid -= bytes;
 955                 } else {
 956                         error = -bytes;
 957                         if (error == ERESTARTSYS)
 958                                 error = ERESTART;
 959                 }
 960         } else
 961                 error = ENXIO;
 962
 963         /* update kqfilter status, if any */
 964         linux_dev_kqfilter_poll(filp, LINUX_KQ_FLAG_HAS_READ);
 965
 966         return (error);
 967 }
 968
 969 static int
 970 linux_dev_write(struct cdev *dev, struct uio *uio, int ioflag)
 971 {
 972         struct linux_file *filp;
 973         struct thread *td;
 974         struct file *file;
 975         ssize_t bytes;
 976         int error;
 977
 978         td = curthread;
 979         file = td->td_fpop;
 980         if (dev->si_drv1 == NULL)
 981                 return (ENXIO);
 982         if ((error = devfs_get_cdevpriv((void **)&filp)) != 0)
 983                 return (error);
 984         filp->f_flags = file->f_flag;
 985         /* XXX no support for I/O vectors currently */
 986         if (uio->uio_iovcnt != 1)
 987                 return (EOPNOTSUPP);
 988         linux_set_current(td);
 989         if (filp->f_op->write) {
 990                 bytes = filp->f_op->write(filp, uio->uio_iov->iov_base,
 991                     uio->uio_iov->iov_len, &uio->uio_offset);
 992                 if (bytes >= 0) {
 993                         uio->uio_iov->iov_base =
 994                             ((uint8_t *)uio->uio_iov->iov_base) + bytes;
 995                         uio->uio_iov->iov_len -= bytes;
 996                         uio->uio_resid -= bytes;
 997                 } else {
 998                         error = -bytes;
 999                         if (error == ERESTARTSYS)
1000                                 error = ERESTART;
1001                 }
1002         } else
1003                 error = ENXIO;
1004
1005         /* update kqfilter status, if any */
1006         linux_dev_kqfilter_poll(filp, LINUX_KQ_FLAG_HAS_WRITE);
1007
1008         return (error);
1009 }
1010
1011 static int
1012 linux_dev_poll(struct cdev *dev, int events, struct thread *td)
1013 {
1014         struct linux_file *filp;
1015         struct file *file;
1016         int revents;
1017
1018         if (dev->si_drv1 == NULL)
1019                 goto error;
1020         if (devfs_get_cdevpriv((void **)&filp) != 0)
1021                 goto error;
1022
1023         file = td->td_fpop;
1024         filp->f_flags = file->f_flag;
1025         linux_set_current(td);
1026         if (filp->f_op->poll != NULL)
1027                 revents = filp->f_op->poll(filp, NULL) & events;
1028         else
1029                 revents = 0;
1030
1031         return (revents);
1032 error:
1033         return (events & (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
1034 }
1035
1036 /*
1037  * This function atomically updates the poll wakeup state and returns
1038  * the previous state at the time of update.
1039  */
1040 static uint8_t
1041 linux_poll_wakeup_state(atomic_t *v, const uint8_t *pstate)
1042 {
1043         int c, old;
1044
1045         c = v->counter;
1046
1047         while ((old = atomic_cmpxchg(v, c, pstate[c])) != c)
1048                 c = old;
1049
1050         return (c);
1051 }
1052
1053
1054 static int
1055 linux_poll_wakeup_callback(wait_queue_t *wq, unsigned int wq_state, int flags, void *key)
1056 {
1057         static const uint8_t state[LINUX_FWQ_STATE_MAX] = {
1058                 [LINUX_FWQ_STATE_INIT] = LINUX_FWQ_STATE_INIT, /* NOP */
1059                 [LINUX_FWQ_STATE_NOT_READY] = LINUX_FWQ_STATE_NOT_READY, /* NOP */
1060                 [LINUX_FWQ_STATE_QUEUED] = LINUX_FWQ_STATE_READY,
1061                 [LINUX_FWQ_STATE_READY] = LINUX_FWQ_STATE_READY, /* NOP */
1062         };
1063         struct linux_file *filp = container_of(wq, struct linux_file, f_wait_queue.wq);
1064
1065         switch (linux_poll_wakeup_state(&filp->f_wait_queue.state, state)) {
1066         case LINUX_FWQ_STATE_QUEUED:
1067                 linux_poll_wakeup(filp);
1068                 return (1);
1069         default:
1070                 return (0);
1071         }
1072 }
1073
1074 void
1075 linux_poll_wait(struct linux_file *filp, wait_queue_head_t *wqh, poll_table *p)
1076 {
1077         static const uint8_t state[LINUX_FWQ_STATE_MAX] = {
1078                 [LINUX_FWQ_STATE_INIT] = LINUX_FWQ_STATE_NOT_READY,
1079                 [LINUX_FWQ_STATE_NOT_READY] = LINUX_FWQ_STATE_NOT_READY, /* NOP */
1080                 [LINUX_FWQ_STATE_QUEUED] = LINUX_FWQ_STATE_QUEUED, /* NOP */
1081                 [LINUX_FWQ_STATE_READY] = LINUX_FWQ_STATE_QUEUED,
1082         };
1083
1084         selrecord(curthread, &filp->f_selinfo);
1085
1086         switch (linux_poll_wakeup_state(&filp->f_wait_queue.state, state)) {
1087         case LINUX_FWQ_STATE_INIT:
1088                 /* NOTE: file handles can only belong to one wait-queue */
1089                 filp->f_wait_queue.wqh = wqh;
1090                 filp->f_wait_queue.wq.func = &linux_poll_wakeup_callback;
1091                 add_wait_queue(wqh, &filp->f_wait_queue.wq);
1092                 atomic_set(&filp->f_wait_queue.state, LINUX_FWQ_STATE_QUEUED);
1093                 break;
1094         default:
1095                 break;
1096         }
1097 }
1098
1099 static void
1100 linux_poll_wait_dequeue(struct linux_file *filp)
1101 {
1102         static const uint8_t state[LINUX_FWQ_STATE_MAX] = {
1103                 [LINUX_FWQ_STATE_INIT] = LINUX_FWQ_STATE_INIT,  /* NOP */
1104                 [LINUX_FWQ_STATE_NOT_READY] = LINUX_FWQ_STATE_INIT,
1105                 [LINUX_FWQ_STATE_QUEUED] = LINUX_FWQ_STATE_INIT,
1106                 [LINUX_FWQ_STATE_READY] = LINUX_FWQ_STATE_INIT,
1107         };
1108
1109         seldrain(&filp->f_selinfo);
1110
1111         switch (linux_poll_wakeup_state(&filp->f_wait_queue.state, state)) {
1112         case LINUX_FWQ_STATE_NOT_READY:
1113         case LINUX_FWQ_STATE_QUEUED:
1114         case LINUX_FWQ_STATE_READY:
1115                 remove_wait_queue(filp->f_wait_queue.wqh, &filp->f_wait_queue.wq);
1116                 break;
1117         default:
1118                 break;
1119         }
1120 }
1121
1122 void
1123 linux_poll_wakeup(struct linux_file *filp)
1124 {
1125         /* this function should be NULL-safe */
1126         if (filp == NULL)
1127                 return;
1128
1129         selwakeup(&filp->f_selinfo);
1130
1131         spin_lock(&filp->f_kqlock);
1132         filp->f_kqflags |= LINUX_KQ_FLAG_NEED_READ |
1133             LINUX_KQ_FLAG_NEED_WRITE;
1134
1135         /* make sure the "knote" gets woken up */
1136         KNOTE_LOCKED(&filp->f_selinfo.si_note, 1);
1137         spin_unlock(&filp->f_kqlock);
1138 }
1139
1140 static void
1141 linux_dev_kqfilter_detach(struct knote *kn)
1142 {
1143         struct linux_file *filp = kn->kn_hook;
1144
1145         spin_lock(&filp->f_kqlock);
1146         knlist_remove(&filp->f_selinfo.si_note, kn, 1);
1147         spin_unlock(&filp->f_kqlock);
1148 }
1149
1150 static int
1151 linux_dev_kqfilter_read_event(struct knote *kn, long hint)
1152 {
1153         struct linux_file *filp = kn->kn_hook;
1154
1155         mtx_assert(&filp->f_kqlock.m, MA_OWNED);
1156
1157         return ((filp->f_kqflags & LINUX_KQ_FLAG_NEED_READ) ? 1 : 0);
1158 }
1159
1160 static int
1161 linux_dev_kqfilter_write_event(struct knote *kn, long hint)
1162 {
1163         struct linux_file *filp = kn->kn_hook;
1164
1165         mtx_assert(&filp->f_kqlock.m, MA_OWNED);
1166
1167         return ((filp->f_kqflags & LINUX_KQ_FLAG_NEED_WRITE) ? 1 : 0);
1168 }
1169
1170 static struct filterops linux_dev_kqfiltops_read = {
1171         .f_isfd = 1,
1172         .f_detach = linux_dev_kqfilter_detach,
1173         .f_event = linux_dev_kqfilter_read_event,
1174 };
1175
1176 static struct filterops linux_dev_kqfiltops_write = {
1177         .f_isfd = 1,
1178         .f_detach = linux_dev_kqfilter_detach,
1179         .f_event = linux_dev_kqfilter_write_event,
1180 };
1181
1182 static void
1183 linux_dev_kqfilter_poll(struct linux_file *filp, int kqflags)
1184 {
1185         int temp;
1186
1187         if (filp->f_kqflags & kqflags) {
1188                 /* get the latest polling state */
1189                 temp = filp->f_op->poll(filp, NULL);
1190
1191                 spin_lock(&filp->f_kqlock);
1192                 /* clear kqflags */
1193                 filp->f_kqflags &= ~(LINUX_KQ_FLAG_NEED_READ |
1194                     LINUX_KQ_FLAG_NEED_WRITE);
1195                 /* update kqflags */
1196                 if (temp & (POLLIN | POLLOUT)) {
1197                         if (temp & POLLIN)
1198                                 filp->f_kqflags |= LINUX_KQ_FLAG_NEED_READ;
1199                         if (temp & POLLOUT)
1200                                 filp->f_kqflags |= LINUX_KQ_FLAG_NEED_WRITE;
1201
1202                         /* make sure the "knote" gets woken up */
1203                         KNOTE_LOCKED(&filp->f_selinfo.si_note, 0);
1204                 }
1205                 spin_unlock(&filp->f_kqlock);
1206         }
1207 }
1208
1209 static int
1210 linux_dev_kqfilter(struct cdev *dev, struct knote *kn)
1211 {
1212         struct linux_file *filp;
1213         struct file *file;
1214         struct thread *td;
1215         int error;
1216
1217         td = curthread;
1218         file = td->td_fpop;
1219         if (dev->si_drv1 == NULL)
1220                 return (ENXIO);
1221         if ((error = devfs_get_cdevpriv((void **)&filp)) != 0)
1222                 return (error);
1223         filp->f_flags = file->f_flag;
1224         if (filp->f_op->poll == NULL)
1225                 return (EINVAL);
1226
1227         spin_lock(&filp->f_kqlock);
1228         switch (kn->kn_filter) {
1229         case EVFILT_READ:
1230                 filp->f_kqflags |= LINUX_KQ_FLAG_HAS_READ;
1231                 kn->kn_fop = &linux_dev_kqfiltops_read;
1232                 kn->kn_hook = filp;
1233                 knlist_add(&filp->f_selinfo.si_note, kn, 1);
1234                 break;
1235         case EVFILT_WRITE:
1236                 filp->f_kqflags |= LINUX_KQ_FLAG_HAS_WRITE;
1237                 kn->kn_fop = &linux_dev_kqfiltops_write;
1238                 kn->kn_hook = filp;
1239                 knlist_add(&filp->f_selinfo.si_note, kn, 1);
1240                 break;
1241         default:
1242                 error = EINVAL;
1243                 break;
1244         }
1245         spin_unlock(&filp->f_kqlock);
1246
1247         if (error == 0) {
1248                 linux_set_current(td);
1249
1250                 /* update kqfilter status, if any */
1251                 linux_dev_kqfilter_poll(filp,
1252                     LINUX_KQ_FLAG_HAS_READ | LINUX_KQ_FLAG_HAS_WRITE);
1253         }
1254         return (error);
1255 }
1256
1257 static int
1258 linux_dev_mmap_single(struct cdev *dev, vm_ooffset_t *offset,
1259     vm_size_t size, struct vm_object **object, int nprot)
1260 {
1261         struct vm_area_struct *vmap;
1262         struct mm_struct *mm;
1263         struct linux_file *filp;
1264         struct thread *td;
1265         struct file *file;
1266         vm_memattr_t attr;
1267         int error;
1268
1269         td = curthread;
1270         file = td->td_fpop;
1271         if (dev->si_drv1 == NULL)
1272                 return (ENODEV);
1273         if ((error = devfs_get_cdevpriv((void **)&filp)) != 0)
1274                 return (error);
1275         filp->f_flags = file->f_flag;
1276
1277         if (filp->f_op->mmap == NULL)
1278                 return (ENODEV);
1279
1280         linux_set_current(td);
1281
1282         /*
1283          * The same VM object might be shared by multiple processes
1284          * and the mm_struct is usually freed when a process exits.
1285          *
1286          * The atomic reference below makes sure the mm_struct is
1287          * available as long as the vmap is in the linux_vma_head.
1288          */
1289         mm = current->mm;
1290         if (atomic_inc_not_zero(&mm->mm_users) == 0)
1291                 return (EINVAL);
1292
1293         vmap = kzalloc(sizeof(*vmap), GFP_KERNEL);
1294         vmap->vm_start = 0;
1295         vmap->vm_end = size;
1296         vmap->vm_pgoff = *offset / PAGE_SIZE;
1297         vmap->vm_pfn = 0;
1298         vmap->vm_flags = vmap->vm_page_prot = (nprot & VM_PROT_ALL);
1299         vmap->vm_ops = NULL;
1300         vmap->vm_file = get_file(filp);
1301         vmap->vm_mm = mm;
1302
1303         if (unlikely(down_write_killable(&vmap->vm_mm->mmap_sem))) {
1304                 error = EINTR;
1305         } else {
1306                 error = -filp->f_op->mmap(filp, vmap);
1307                 up_write(&vmap->vm_mm->mmap_sem);
1308         }
1309
1310         if (error != 0) {
1311                 linux_cdev_handle_free(vmap);
1312                 return (error);
1313         }
1314
1315         attr = pgprot2cachemode(vmap->vm_page_prot);
1316
1317         if (vmap->vm_ops != NULL) {
1318                 void *vm_private_data;
1319
1320                 if (vmap->vm_ops->open == NULL ||
1321                     vmap->vm_ops->close == NULL ||
1322                     vmap->vm_private_data == NULL) {
1323                         linux_cdev_handle_free(vmap);
1324                         return (EINVAL);
1325                 }
1326
1327                 vm_private_data = vmap->vm_private_data;
1328
1329                 vmap = linux_cdev_handle_insert(vm_private_data, vmap);
1330
1331                 if (vmap->vm_ops->fault == NULL) {
1332                         *object = cdev_pager_allocate(vm_private_data, OBJT_DEVICE,
1333                             &linux_cdev_pager_ops[1], size, nprot, *offset,
1334                             curthread->td_ucred);
1335                 } else {
1336                         *object = cdev_pager_allocate(vm_private_data, OBJT_MGTDEVICE,
1337                             &linux_cdev_pager_ops[0], size, nprot, *offset,
1338                             curthread->td_ucred);
1339                 }
1340
1341                 if (*object == NULL) {
1342                         linux_cdev_handle_remove(vmap);
1343                         linux_cdev_handle_free(vmap);
1344                         return (EINVAL);
1345                 }
1346         } else {
1347                 struct sglist *sg;
1348
1349                 sg = sglist_alloc(1, M_WAITOK);
1350                 sglist_append_phys(sg,
1351                     (vm_paddr_t)vmap->vm_pfn << PAGE_SHIFT, vmap->vm_len);
1352
1353                 *object = vm_pager_allocate(OBJT_SG, sg, vmap->vm_len,
1354                     nprot, 0, curthread->td_ucred);
1355
1356                 linux_cdev_handle_free(vmap);
1357
1358                 if (*object == NULL) {
1359                         sglist_free(sg);
1360                         return (EINVAL);
1361                 }
1362         }
1363
1364         if (attr != VM_MEMATTR_DEFAULT) {
1365                 VM_OBJECT_WLOCK(*object);
1366                 vm_object_set_memattr(*object, attr);
1367                 VM_OBJECT_WUNLOCK(*object);
1368         }
1369         *offset = 0;
1370         return (0);
1371 }
1372
1373 struct cdevsw linuxcdevsw = {
1374         .d_version = D_VERSION,
1375         .d_flags = D_TRACKCLOSE,
1376         .d_open = linux_dev_open,
1377         .d_close = linux_dev_close,
1378         .d_read = linux_dev_read,
1379         .d_write = linux_dev_write,
1380         .d_ioctl = linux_dev_ioctl,
1381         .d_mmap_single = linux_dev_mmap_single,
1382         .d_poll = linux_dev_poll,
1383         .d_kqfilter = linux_dev_kqfilter,
1384         .d_name = "lkpidev",
1385 };
1386
1387 static int
1388 linux_file_read(struct file *file, struct uio *uio, struct ucred *active_cred,
1389     int flags, struct thread *td)
1390 {
1391         struct linux_file *filp;
1392         ssize_t bytes;
1393         int error;
1394
1395         error = 0;
1396         filp = (struct linux_file *)file->f_data;
1397         filp->f_flags = file->f_flag;
1398         /* XXX no support for I/O vectors currently */
1399         if (uio->uio_iovcnt != 1)
1400                 return (EOPNOTSUPP);
1401         linux_set_current(td);
1402         if (filp->f_op->read) {
1403                 bytes = filp->f_op->read(filp, uio->uio_iov->iov_base,
1404                     uio->uio_iov->iov_len, &uio->uio_offset);
1405                 if (bytes >= 0) {
1406                         uio->uio_iov->iov_base =
1407                             ((uint8_t *)uio->uio_iov->iov_base) + bytes;
1408                         uio->uio_iov->iov_len -= bytes;
1409                         uio->uio_resid -= bytes;
1410                 } else
1411                         error = -bytes;
1412         } else
1413                 error = ENXIO;
1414
1415         return (error);
1416 }
1417
1418 static int
1419 linux_file_poll(struct file *file, int events, struct ucred *active_cred,
1420     struct thread *td)
1421 {
1422         struct linux_file *filp;
1423         int revents;
1424
1425         filp = (struct linux_file *)file->f_data;
1426         filp->f_flags = file->f_flag;
1427         linux_set_current(td);
1428         if (filp->f_op->poll != NULL) {
1429                 selrecord(td, &filp->f_selinfo);
1430                 revents = filp->f_op->poll(filp, NULL) & events;
1431         } else
1432                 revents = 0;
1433
1434         return (revents);
1435 }
1436
1437 static int
1438 linux_file_close(struct file *file, struct thread *td)
1439 {
1440         struct linux_file *filp;
1441         int error;
1442
1443         filp = (struct linux_file *)file->f_data;
1444         filp->f_flags = file->f_flag;
1445         linux_set_current(td);
1446         linux_poll_wait_dequeue(filp);
1447         error = -filp->f_op->release(NULL, filp);
1448         funsetown(&filp->f_sigio);
1449         kfree(filp);
1450
1451         return (error);
1452 }
1453
1454 static int
1455 linux_file_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *cred,
1456     struct thread *td)
1457 {
1458         struct linux_file *filp;
1459         int error;
1460
1461         filp = (struct linux_file *)fp->f_data;
1462         filp->f_flags = fp->f_flag;
1463         error = 0;
1464
1465         linux_set_current(td);
1466         switch (cmd) {
1467         case FIONBIO:
1468                 break;
1469         case FIOASYNC:
1470                 if (filp->f_op->fasync == NULL)
1471                         break;
1472                 error = filp->f_op->fasync(0, filp, fp->f_flag & FASYNC);
1473                 break;
1474         case FIOSETOWN:
1475                 error = fsetown(*(int *)data, &filp->f_sigio);
1476                 if (error == 0)
1477                         error = filp->f_op->fasync(0, filp,
1478                             fp->f_flag & FASYNC);
1479                 break;
1480         case FIOGETOWN:
1481                 *(int *)data = fgetown(&filp->f_sigio);
1482                 break;
1483         default:
1484                 error = ENOTTY;
1485                 break;
1486         }
1487         return (error);
1488 }
1489
1490 static int
1491 linux_file_stat(struct file *fp, struct stat *sb, struct ucred *active_cred,
1492     struct thread *td)
1493 {
1494
1495         return (EOPNOTSUPP);
1496 }
1497
1498 static int
1499 linux_file_fill_kinfo(struct file *fp, struct kinfo_file *kif,
1500     struct filedesc *fdp)
1501 {
1502
1503         return (0);
1504 }
1505
1506 unsigned int
1507 linux_iminor(struct inode *inode)
1508 {
1509         struct linux_cdev *ldev;
1510
1511         if (inode == NULL || inode->v_rdev == NULL ||
1512             inode->v_rdev->si_devsw != &linuxcdevsw)
1513                 return (-1U);
1514         ldev = inode->v_rdev->si_drv1;
1515         if (ldev == NULL)
1516                 return (-1U);
1517
1518         return (minor(ldev->dev));
1519 }
1520
1521 struct fileops linuxfileops = {
1522         .fo_read = linux_file_read,
1523         .fo_write = invfo_rdwr,
1524         .fo_truncate = invfo_truncate,
1525         .fo_kqfilter = invfo_kqfilter,
1526         .fo_stat = linux_file_stat,
1527         .fo_fill_kinfo = linux_file_fill_kinfo,
1528         .fo_poll = linux_file_poll,
1529         .fo_close = linux_file_close,
1530         .fo_ioctl = linux_file_ioctl,
1531         .fo_chmod = invfo_chmod,
1532         .fo_chown = invfo_chown,
1533         .fo_sendfile = invfo_sendfile,
1534 };
1535
1536 /*
1537  * Hash of vmmap addresses.  This is infrequently accessed and does not
1538  * need to be particularly large.  This is done because we must store the
1539  * caller's idea of the map size to properly unmap.
1540  */
1541 struct vmmap {
1542         LIST_ENTRY(vmmap)       vm_next;
1543         void                    *vm_addr;
1544         unsigned long           vm_size;
1545 };
1546
1547 struct vmmaphd {
1548         struct vmmap *lh_first;
1549 };
1550 #define VMMAP_HASH_SIZE 64
1551 #define VMMAP_HASH_MASK (VMMAP_HASH_SIZE - 1)
1552 #define VM_HASH(addr)   ((uintptr_t)(addr) >> PAGE_SHIFT) & VMMAP_HASH_MASK
1553 static struct vmmaphd vmmaphead[VMMAP_HASH_SIZE];
1554 static struct mtx vmmaplock;
1555
1556 static void
1557 vmmap_add(void *addr, unsigned long size)
1558 {
1559         struct vmmap *vmmap;
1560
1561         vmmap = kmalloc(sizeof(*vmmap), GFP_KERNEL);
1562         mtx_lock(&vmmaplock);
1563         vmmap->vm_size = size;
1564         vmmap->vm_addr = addr;
1565         LIST_INSERT_HEAD(&vmmaphead[VM_HASH(addr)], vmmap, vm_next);
1566         mtx_unlock(&vmmaplock);
1567 }
1568
1569 static struct vmmap *
1570 vmmap_remove(void *addr)
1571 {
1572         struct vmmap *vmmap;
1573
1574         mtx_lock(&vmmaplock);
1575         LIST_FOREACH(vmmap, &vmmaphead[VM_HASH(addr)], vm_next)
1576                 if (vmmap->vm_addr == addr)
1577                         break;
1578         if (vmmap)
1579                 LIST_REMOVE(vmmap, vm_next);
1580         mtx_unlock(&vmmaplock);
1581
1582         return (vmmap);
1583 }
1584
1585 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
1586 void *
1587 _ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr)
1588 {
1589         void *addr;
1590
1591         addr = pmap_mapdev_attr(phys_addr, size, attr);
1592         if (addr == NULL)
1593                 return (NULL);
1594         vmmap_add(addr, size);
1595
1596         return (addr);
1597 }
1598 #endif
1599
1600 void
1601 iounmap(void *addr)
1602 {
1603         struct vmmap *vmmap;
1604
1605         vmmap = vmmap_remove(addr);
1606         if (vmmap == NULL)
1607                 return;
1608 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
1609         pmap_unmapdev((vm_offset_t)addr, vmmap->vm_size);
1610 #endif
1611         kfree(vmmap);
1612 }
1613
1614
1615 void *
1616 vmap(struct page **pages, unsigned int count, unsigned long flags, int prot)
1617 {
1618         vm_offset_t off;
1619         size_t size;
1620
1621         size = count * PAGE_SIZE;
1622         off = kva_alloc(size);
1623         if (off == 0)
1624                 return (NULL);
1625         vmmap_add((void *)off, size);
1626         pmap_qenter(off, pages, count);
1627
1628         return ((void *)off);
1629 }
1630
1631 void
1632 vunmap(void *addr)
1633 {
1634         struct vmmap *vmmap;
1635
1636         vmmap = vmmap_remove(addr);
1637         if (vmmap == NULL)
1638                 return;
1639         pmap_qremove((vm_offset_t)addr, vmmap->vm_size / PAGE_SIZE);
1640         kva_free((vm_offset_t)addr, vmmap->vm_size);
1641         kfree(vmmap);
1642 }
1643
1644 char *
1645 kvasprintf(gfp_t gfp, const char *fmt, va_list ap)
1646 {
1647         unsigned int len;
1648         char *p;
1649         va_list aq;
1650
1651         va_copy(aq, ap);
1652         len = vsnprintf(NULL, 0, fmt, aq);
1653         va_end(aq);
1654
1655         p = kmalloc(len + 1, gfp);
1656         if (p != NULL)
1657                 vsnprintf(p, len + 1, fmt, ap);
1658
1659         return (p);
1660 }
1661
1662 char *
1663 kasprintf(gfp_t gfp, const char *fmt, ...)
1664 {
1665         va_list ap;
1666         char *p;
1667
1668         va_start(ap, fmt);
1669         p = kvasprintf(gfp, fmt, ap);
1670         va_end(ap);
1671
1672         return (p);
1673 }
1674
1675 static void
1676 linux_timer_callback_wrapper(void *context)
1677 {
1678         struct timer_list *timer;
1679
1680         linux_set_current(curthread);
1681
1682         timer = context;
1683         timer->function(timer->data);
1684 }
1685
1686 void
1687 mod_timer(struct timer_list *timer, int expires)
1688 {
1689
1690         timer->expires = expires;
1691         callout_reset(&timer->timer_callout,
1692             linux_timer_jiffies_until(expires),
1693             &linux_timer_callback_wrapper, timer);
1694 }
1695
1696 void
1697 add_timer(struct timer_list *timer)
1698 {
1699
1700         callout_reset(&timer->timer_callout,
1701             linux_timer_jiffies_until(timer->expires),
1702             &linux_timer_callback_wrapper, timer);
1703 }
1704
1705 void
1706 add_timer_on(struct timer_list *timer, int cpu)
1707 {
1708
1709         callout_reset_on(&timer->timer_callout,
1710             linux_timer_jiffies_until(timer->expires),
1711             &linux_timer_callback_wrapper, timer, cpu);
1712 }
1713
1714 static void
1715 linux_timer_init(void *arg)
1716 {
1717
1718         /*
1719          * Compute an internal HZ value which can divide 2**32 to
1720          * avoid timer rounding problems when the tick value wraps
1721          * around 2**32:
1722          */
1723         linux_timer_hz_mask = 1;
1724         while (linux_timer_hz_mask < (unsigned long)hz)
1725                 linux_timer_hz_mask *= 2;
1726         linux_timer_hz_mask--;
1727 }
1728 SYSINIT(linux_timer, SI_SUB_DRIVERS, SI_ORDER_FIRST, linux_timer_init, NULL);
1729
1730 void
1731 linux_complete_common(struct completion *c, int all)
1732 {
1733         int wakeup_swapper;
1734
1735         sleepq_lock(c);
1736         c->done++;
1737         if (all)
1738                 wakeup_swapper = sleepq_broadcast(c, SLEEPQ_SLEEP, 0, 0);
1739         else
1740                 wakeup_swapper = sleepq_signal(c, SLEEPQ_SLEEP, 0, 0);
1741         sleepq_release(c);
1742         if (wakeup_swapper)
1743                 kick_proc0();
1744 }
1745
1746 /*
1747  * Indefinite wait for done != 0 with or without signals.
1748  */
1749 int
1750 linux_wait_for_common(struct completion *c, int flags)
1751 {
1752         int error;
1753
1754         if (SCHEDULER_STOPPED())
1755                 return (0);
1756
1757         DROP_GIANT();
1758
1759         if (flags != 0)
1760                 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP;
1761         else
1762                 flags = SLEEPQ_SLEEP;
1763         error = 0;
1764         for (;;) {
1765                 sleepq_lock(c);
1766                 if (c->done)
1767                         break;
1768                 sleepq_add(c, NULL, "completion", flags, 0);
1769                 if (flags & SLEEPQ_INTERRUPTIBLE) {
1770                         if (sleepq_wait_sig(c, 0) != 0) {
1771                                 error = -ERESTARTSYS;
1772                                 goto intr;
1773                         }
1774                 } else
1775                         sleepq_wait(c, 0);
1776         }
1777         c->done--;
1778         sleepq_release(c);
1779
1780 intr:
1781         PICKUP_GIANT();
1782
1783         return (error);
1784 }
1785
1786 /*
1787  * Time limited wait for done != 0 with or without signals.
1788  */
1789 int
1790 linux_wait_for_timeout_common(struct completion *c, int timeout, int flags)
1791 {
1792         int end = jiffies + timeout;
1793         int error;
1794         int ret;
1795
1796         if (SCHEDULER_STOPPED())
1797                 return (0);
1798
1799         DROP_GIANT();
1800
1801         if (flags != 0)
1802                 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP;
1803         else
1804                 flags = SLEEPQ_SLEEP;
1805
1806         error = 0;
1807         ret = 0;
1808         for (;;) {
1809                 sleepq_lock(c);
1810                 if (c->done)
1811                         break;
1812                 sleepq_add(c, NULL, "completion", flags, 0);
1813                 sleepq_set_timeout(c, linux_timer_jiffies_until(end));
1814                 if (flags & SLEEPQ_INTERRUPTIBLE)
1815                         ret = sleepq_timedwait_sig(c, 0);
1816                 else
1817                         ret = sleepq_timedwait(c, 0);
1818                 if (ret != 0) {
1819                         /* check for timeout or signal */
1820                         if (ret == EWOULDBLOCK)
1821                                 error = 0;
1822                         else
1823                                 error = -ERESTARTSYS;
1824                         goto intr;
1825                 }
1826         }
1827         c->done--;
1828         sleepq_release(c);
1829
1830 intr:
1831         PICKUP_GIANT();
1832
1833         /* return how many jiffies are left */
1834         return (ret != 0 ? error : linux_timer_jiffies_until(end));
1835 }
1836
1837 int
1838 linux_try_wait_for_completion(struct completion *c)
1839 {
1840         int isdone;
1841
1842         isdone = 1;
1843         sleepq_lock(c);
1844         if (c->done)
1845                 c->done--;
1846         else
1847                 isdone = 0;
1848         sleepq_release(c);
1849         return (isdone);
1850 }
1851
1852 int
1853 linux_completion_done(struct completion *c)
1854 {
1855         int isdone;
1856
1857         isdone = 1;
1858         sleepq_lock(c);
1859         if (c->done == 0)
1860                 isdone = 0;
1861         sleepq_release(c);
1862         return (isdone);
1863 }
1864
1865 static void
1866 linux_cdev_release(struct kobject *kobj)
1867 {
1868         struct linux_cdev *cdev;
1869         struct kobject *parent;
1870
1871         cdev = container_of(kobj, struct linux_cdev, kobj);
1872         parent = kobj->parent;
1873         if (cdev->cdev)
1874                 destroy_dev(cdev->cdev);
1875         kfree(cdev);
1876         kobject_put(parent);
1877 }
1878
1879 static void
1880 linux_cdev_static_release(struct kobject *kobj)
1881 {
1882         struct linux_cdev *cdev;
1883         struct kobject *parent;
1884
1885         cdev = container_of(kobj, struct linux_cdev, kobj);
1886         parent = kobj->parent;
1887         if (cdev->cdev)
1888                 destroy_dev(cdev->cdev);
1889         kobject_put(parent);
1890 }
1891
1892 const struct kobj_type linux_cdev_ktype = {
1893         .release = linux_cdev_release,
1894 };
1895
1896 const struct kobj_type linux_cdev_static_ktype = {
1897         .release = linux_cdev_static_release,
1898 };
1899
1900 static void
1901 linux_handle_ifnet_link_event(void *arg, struct ifnet *ifp, int linkstate)
1902 {
1903         struct notifier_block *nb;
1904
1905         nb = arg;
1906         if (linkstate == LINK_STATE_UP)
1907                 nb->notifier_call(nb, NETDEV_UP, ifp);
1908         else
1909                 nb->notifier_call(nb, NETDEV_DOWN, ifp);
1910 }
1911
1912 static void
1913 linux_handle_ifnet_arrival_event(void *arg, struct ifnet *ifp)
1914 {
1915         struct notifier_block *nb;
1916
1917         nb = arg;
1918         nb->notifier_call(nb, NETDEV_REGISTER, ifp);
1919 }
1920
1921 static void
1922 linux_handle_ifnet_departure_event(void *arg, struct ifnet *ifp)
1923 {
1924         struct notifier_block *nb;
1925
1926         nb = arg;
1927         nb->notifier_call(nb, NETDEV_UNREGISTER, ifp);
1928 }
1929
1930 static void
1931 linux_handle_iflladdr_event(void *arg, struct ifnet *ifp)
1932 {
1933         struct notifier_block *nb;
1934
1935         nb = arg;
1936         nb->notifier_call(nb, NETDEV_CHANGEADDR, ifp);
1937 }
1938
1939 static void
1940 linux_handle_ifaddr_event(void *arg, struct ifnet *ifp)
1941 {
1942         struct notifier_block *nb;
1943
1944         nb = arg;
1945         nb->notifier_call(nb, NETDEV_CHANGEIFADDR, ifp);
1946 }
1947
1948 int
1949 register_netdevice_notifier(struct notifier_block *nb)
1950 {
1951
1952         nb->tags[NETDEV_UP] = EVENTHANDLER_REGISTER(
1953             ifnet_link_event, linux_handle_ifnet_link_event, nb, 0);
1954         nb->tags[NETDEV_REGISTER] = EVENTHANDLER_REGISTER(
1955             ifnet_arrival_event, linux_handle_ifnet_arrival_event, nb, 0);
1956         nb->tags[NETDEV_UNREGISTER] = EVENTHANDLER_REGISTER(
1957             ifnet_departure_event, linux_handle_ifnet_departure_event, nb, 0);
1958         nb->tags[NETDEV_CHANGEADDR] = EVENTHANDLER_REGISTER(
1959             iflladdr_event, linux_handle_iflladdr_event, nb, 0);
1960
1961         return (0);
1962 }
1963
1964 int
1965 register_inetaddr_notifier(struct notifier_block *nb)
1966 {
1967
1968         nb->tags[NETDEV_CHANGEIFADDR] = EVENTHANDLER_REGISTER(
1969             ifaddr_event, linux_handle_ifaddr_event, nb, 0);
1970         return (0);
1971 }
1972
1973 int
1974 unregister_netdevice_notifier(struct notifier_block *nb)
1975 {
1976
1977         EVENTHANDLER_DEREGISTER(ifnet_link_event,
1978             nb->tags[NETDEV_UP]);
1979         EVENTHANDLER_DEREGISTER(ifnet_arrival_event,
1980             nb->tags[NETDEV_REGISTER]);
1981         EVENTHANDLER_DEREGISTER(ifnet_departure_event,
1982             nb->tags[NETDEV_UNREGISTER]);
1983         EVENTHANDLER_DEREGISTER(iflladdr_event,
1984             nb->tags[NETDEV_CHANGEADDR]);
1985
1986         return (0);
1987 }
1988
1989 int
1990 unregister_inetaddr_notifier(struct notifier_block *nb)
1991 {
1992
1993         EVENTHANDLER_DEREGISTER(ifaddr_event,
1994             nb->tags[NETDEV_CHANGEIFADDR]);
1995
1996         return (0);
1997 }
1998
1999 struct list_sort_thunk {
2000         int (*cmp)(void *, struct list_head *, struct list_head *);
2001         void *priv;
2002 };
2003
2004 static inline int
2005 linux_le_cmp(void *priv, const void *d1, const void *d2)
2006 {
2007         struct list_head *le1, *le2;
2008         struct list_sort_thunk *thunk;
2009
2010         thunk = priv;
2011         le1 = *(__DECONST(struct list_head **, d1));
2012         le2 = *(__DECONST(struct list_head **, d2));
2013         return ((thunk->cmp)(thunk->priv, le1, le2));
2014 }
2015
2016 void
2017 list_sort(void *priv, struct list_head *head, int (*cmp)(void *priv,
2018     struct list_head *a, struct list_head *b))
2019 {
2020         struct list_sort_thunk thunk;
2021         struct list_head **ar, *le;
2022         size_t count, i;
2023
2024         count = 0;
2025         list_for_each(le, head)
2026                 count++;
2027         ar = malloc(sizeof(struct list_head *) * count, M_KMALLOC, M_WAITOK);
2028         i = 0;
2029         list_for_each(le, head)
2030                 ar[i++] = le;
2031         thunk.cmp = cmp;
2032         thunk.priv = priv;
2033         qsort_r(ar, count, sizeof(struct list_head *), &thunk, linux_le_cmp);
2034         INIT_LIST_HEAD(head);
2035         for (i = 0; i < count; i++)
2036                 list_add_tail(ar[i], head);
2037         free(ar, M_KMALLOC);
2038 }
2039
2040 void
2041 linux_irq_handler(void *ent)
2042 {
2043         struct irq_ent *irqe;
2044
2045         linux_set_current(curthread);
2046
2047         irqe = ent;
2048         irqe->handler(irqe->irq, irqe->arg);
2049 }
2050
2051 #if defined(__i386__) || defined(__amd64__)
2052 int
2053 linux_wbinvd_on_all_cpus(void)
2054 {
2055
2056         pmap_invalidate_cache();
2057         return (0);
2058 }
2059 #endif
2060
2061 int
2062 linux_on_each_cpu(void callback(void *), void *data)
2063 {
2064
2065         smp_rendezvous(smp_no_rendezvous_barrier, callback,
2066             smp_no_rendezvous_barrier, data);
2067         return (0);
2068 }
2069
2070 int
2071 linux_in_atomic(void)
2072 {
2073
2074         return ((curthread->td_pflags & TDP_NOFAULTING) != 0);
2075 }
2076
2077 struct linux_cdev *
2078 linux_find_cdev(const char *name, unsigned major, unsigned minor)
2079 {
2080         dev_t dev = MKDEV(major, minor);
2081         struct cdev *cdev;
2082
2083         dev_lock();
2084         LIST_FOREACH(cdev, &linuxcdevsw.d_devs, si_list) {
2085                 struct linux_cdev *ldev = cdev->si_drv1;
2086                 if (ldev->dev == dev &&
2087                     strcmp(kobject_name(&ldev->kobj), name) == 0) {
2088                         break;
2089                 }
2090         }
2091         dev_unlock();
2092
2093         return (cdev != NULL ? cdev->si_drv1 : NULL);
2094 }
2095
2096 int
2097 __register_chrdev(unsigned int major, unsigned int baseminor,
2098     unsigned int count, const char *name,
2099     const struct file_operations *fops)
2100 {
2101         struct linux_cdev *cdev;
2102         int ret = 0;
2103         int i;
2104
2105         for (i = baseminor; i < baseminor + count; i++) {
2106                 cdev = cdev_alloc();
2107                 cdev_init(cdev, fops);
2108                 kobject_set_name(&cdev->kobj, name);
2109
2110                 ret = cdev_add(cdev, makedev(major, i), 1);
2111                 if (ret != 0)
2112                         break;
2113         }
2114         return (ret);
2115 }
2116
2117 int
2118 __register_chrdev_p(unsigned int major, unsigned int baseminor,
2119     unsigned int count, const char *name,
2120     const struct file_operations *fops, uid_t uid,
2121     gid_t gid, int mode)
2122 {
2123         struct linux_cdev *cdev;
2124         int ret = 0;
2125         int i;
2126
2127         for (i = baseminor; i < baseminor + count; i++) {
2128                 cdev = cdev_alloc();
2129                 cdev_init(cdev, fops);
2130                 kobject_set_name(&cdev->kobj, name);
2131
2132                 ret = cdev_add_ext(cdev, makedev(major, i), uid, gid, mode);
2133                 if (ret != 0)
2134                         break;
2135         }
2136         return (ret);
2137 }
2138
2139 void
2140 __unregister_chrdev(unsigned int major, unsigned int baseminor,
2141     unsigned int count, const char *name)
2142 {
2143         struct linux_cdev *cdevp;
2144         int i;
2145
2146         for (i = baseminor; i < baseminor + count; i++) {
2147                 cdevp = linux_find_cdev(name, major, i);
2148                 if (cdevp != NULL)
2149                         cdev_del(cdevp);
2150         }
2151 }
2152
2153 #if defined(__i386__) || defined(__amd64__)
2154 bool linux_cpu_has_clflush;
2155 #endif
2156
2157 static void
2158 linux_compat_init(void *arg)
2159 {
2160         struct sysctl_oid *rootoid;
2161         int i;
2162
2163 #if defined(__i386__) || defined(__amd64__)
2164         linux_cpu_has_clflush = (cpu_feature & CPUID_CLFSH);
2165 #endif
2166         rw_init(&linux_vma_lock, "lkpi-vma-lock");
2167
2168         rootoid = SYSCTL_ADD_ROOT_NODE(NULL,
2169             OID_AUTO, "sys", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "sys");
2170         kobject_init(&linux_class_root, &linux_class_ktype);
2171         kobject_set_name(&linux_class_root, "class");
2172         linux_class_root.oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(rootoid),
2173             OID_AUTO, "class", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "class");
2174         kobject_init(&linux_root_device.kobj, &linux_dev_ktype);
2175         kobject_set_name(&linux_root_device.kobj, "device");
2176         linux_root_device.kobj.oidp = SYSCTL_ADD_NODE(NULL,
2177             SYSCTL_CHILDREN(rootoid), OID_AUTO, "device", CTLFLAG_RD, NULL,
2178             "device");
2179         linux_root_device.bsddev = root_bus;
2180         linux_class_misc.name = "misc";
2181         class_register(&linux_class_misc);
2182         INIT_LIST_HEAD(&pci_drivers);
2183         INIT_LIST_HEAD(&pci_devices);
2184         spin_lock_init(&pci_lock);
2185         mtx_init(&vmmaplock, "IO Map lock", NULL, MTX_DEF);
2186         for (i = 0; i < VMMAP_HASH_SIZE; i++)
2187                 LIST_INIT(&vmmaphead[i]);
2188 }
2189 SYSINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_init, NULL);
2190
2191 static void
2192 linux_compat_uninit(void *arg)
2193 {
2194         linux_kobject_kfree_name(&linux_class_root);
2195         linux_kobject_kfree_name(&linux_root_device.kobj);
2196         linux_kobject_kfree_name(&linux_class_misc.kobj);
2197
2198         mtx_destroy(&vmmaplock);
2199         spin_lock_destroy(&pci_lock);
2200         rw_destroy(&linux_vma_lock);
2201 }
2202 SYSUNINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_uninit, NULL);
2203
2204 /*
2205  * NOTE: Linux frequently uses "unsigned long" for pointer to integer
2206  * conversion and vice versa, where in FreeBSD "uintptr_t" would be
2207  * used. Assert these types have the same size, else some parts of the
2208  * LinuxKPI may not work like expected:
2209  */
2210 CTASSERT(sizeof(unsigned long) == sizeof(uintptr_t));