sys/compat/ndis/subr_ntoskrnl.c

   1 /*-
   2  * Copyright (c) 2003
   3  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  * 3. All advertising materials mentioning features or use of this software
  14  *    must display the following acknowledgement:
  15  *      This product includes software developed by Bill Paul.
  16  * 4. Neither the name of the author nor the names of any co-contributors
  17  *    may be used to endorse or promote products derived from this software
  18  *    without specific prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  30  * THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include <sys/ctype.h>
  37 #include <sys/unistd.h>
  38 #include <sys/param.h>
  39 #include <sys/types.h>
  40 #include <sys/errno.h>
  41 #include <sys/systm.h>
  42 #include <sys/malloc.h>
  43 #include <sys/lock.h>
  44 #include <sys/mutex.h>
  45
  46 #include <sys/callout.h>
  47 #include <sys/kdb.h>
  48 #include <sys/kernel.h>
  49 #include <sys/proc.h>
  50 #include <sys/condvar.h>
  51 #include <sys/kthread.h>
  52 #include <sys/module.h>
  53 #include <sys/smp.h>
  54 #include <sys/sched.h>
  55 #include <sys/sysctl.h>
  56
  57 #include <machine/atomic.h>
  58 #include <machine/bus.h>
  59 #include <machine/stdarg.h>
  60 #include <machine/resource.h>
  61
  62 #include <sys/bus.h>
  63 #include <sys/rman.h>
  64
  65 #include <vm/vm.h>
  66 #include <vm/vm_param.h>
  67 #include <vm/pmap.h>
  68 #include <vm/uma.h>
  69 #include <vm/vm_kern.h>
  70 #include <vm/vm_map.h>
  71 #include <vm/vm_extern.h>
  72
  73 #include <compat/ndis/pe_var.h>
  74 #include <compat/ndis/cfg_var.h>
  75 #include <compat/ndis/resource_var.h>
  76 #include <compat/ndis/ntoskrnl_var.h>
  77 #include <compat/ndis/hal_var.h>
  78 #include <compat/ndis/ndis_var.h>
  79
  80 #ifdef NTOSKRNL_DEBUG_TIMERS
  81 static int sysctl_show_timers(SYSCTL_HANDLER_ARGS);
  82
  83 SYSCTL_PROC(_debug, OID_AUTO, ntoskrnl_timers, CTLFLAG_RW, 0, 0,
  84         sysctl_show_timers, "I", "Show ntoskrnl timer stats");
  85 #endif
  86
  87 struct kdpc_queue {
  88         list_entry              kq_disp;
  89         struct thread           *kq_td;
  90         int                     kq_cpu;
  91         int                     kq_exit;
  92         int                     kq_running;
  93         kspin_lock              kq_lock;
  94         nt_kevent               kq_proc;
  95         nt_kevent               kq_done;
  96 };
  97
  98 typedef struct kdpc_queue kdpc_queue;
  99
 100 struct wb_ext {
 101         struct cv               we_cv;
 102         struct thread           *we_td;
 103 };
 104
 105 typedef struct wb_ext wb_ext;
 106
 107 #define NTOSKRNL_TIMEOUTS       256
 108 #ifdef NTOSKRNL_DEBUG_TIMERS
 109 static uint64_t ntoskrnl_timer_fires;
 110 static uint64_t ntoskrnl_timer_sets;
 111 static uint64_t ntoskrnl_timer_reloads;
 112 static uint64_t ntoskrnl_timer_cancels;
 113 #endif
 114
 115 struct callout_entry {
 116         struct callout          ce_callout;
 117         list_entry              ce_list;
 118 };
 119
 120 typedef struct callout_entry callout_entry;
 121
 122 static struct list_entry ntoskrnl_calllist;
 123 static struct mtx ntoskrnl_calllock;
 124 struct kuser_shared_data kuser_shared_data;
 125
 126 static struct list_entry ntoskrnl_intlist;
 127 static kspin_lock ntoskrnl_intlock;
 128
 129 static uint8_t RtlEqualUnicodeString(unicode_string *,
 130         unicode_string *, uint8_t);
 131 static void RtlCopyString(ansi_string *, const ansi_string *);
 132 static void RtlCopyUnicodeString(unicode_string *,
 133         unicode_string *);
 134 static irp *IoBuildSynchronousFsdRequest(uint32_t, device_object *,
 135          void *, uint32_t, uint64_t *, nt_kevent *, io_status_block *);
 136 static irp *IoBuildAsynchronousFsdRequest(uint32_t,
 137         device_object *, void *, uint32_t, uint64_t *, io_status_block *);
 138 static irp *IoBuildDeviceIoControlRequest(uint32_t,
 139         device_object *, void *, uint32_t, void *, uint32_t,
 140         uint8_t, nt_kevent *, io_status_block *);
 141 static irp *IoAllocateIrp(uint8_t, uint8_t);
 142 static void IoReuseIrp(irp *, uint32_t);
 143 static void IoFreeIrp(irp *);
 144 static void IoInitializeIrp(irp *, uint16_t, uint8_t);
 145 static irp *IoMakeAssociatedIrp(irp *, uint8_t);
 146 static uint32_t KeWaitForMultipleObjects(uint32_t,
 147         nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t,
 148         int64_t *, wait_block *);
 149 static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t);
 150 static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *);
 151 static void ntoskrnl_satisfy_multiple_waits(wait_block *);
 152 static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *);
 153 static void ntoskrnl_insert_timer(ktimer *, int);
 154 static void ntoskrnl_remove_timer(ktimer *);
 155 #ifdef NTOSKRNL_DEBUG_TIMERS
 156 static void ntoskrnl_show_timers(void);
 157 #endif
 158 static void ntoskrnl_timercall(void *);
 159 static void ntoskrnl_dpc_thread(void *);
 160 static void ntoskrnl_destroy_dpc_threads(void);
 161 static void ntoskrnl_destroy_workitem_threads(void);
 162 static void ntoskrnl_workitem_thread(void *);
 163 static void ntoskrnl_workitem(device_object *, void *);
 164 static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int);
 165 static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int);
 166 static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *);
 167 static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t);
 168 static uint16_t READ_REGISTER_USHORT(uint16_t *);
 169 static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t);
 170 static uint32_t READ_REGISTER_ULONG(uint32_t *);
 171 static void WRITE_REGISTER_UCHAR(uint8_t *, uint8_t);
 172 static uint8_t READ_REGISTER_UCHAR(uint8_t *);
 173 static int64_t _allmul(int64_t, int64_t);
 174 static int64_t _alldiv(int64_t, int64_t);
 175 static int64_t _allrem(int64_t, int64_t);
 176 static int64_t _allshr(int64_t, uint8_t);
 177 static int64_t _allshl(int64_t, uint8_t);
 178 static uint64_t _aullmul(uint64_t, uint64_t);
 179 static uint64_t _aulldiv(uint64_t, uint64_t);
 180 static uint64_t _aullrem(uint64_t, uint64_t);
 181 static uint64_t _aullshr(uint64_t, uint8_t);
 182 static uint64_t _aullshl(uint64_t, uint8_t);
 183 static slist_entry *ntoskrnl_pushsl(slist_header *, slist_entry *);
 184 static void InitializeSListHead(slist_header *);
 185 static slist_entry *ntoskrnl_popsl(slist_header *);
 186 static void ExFreePoolWithTag(void *, uint32_t);
 187 static void ExInitializePagedLookasideList(paged_lookaside_list *,
 188         lookaside_alloc_func *, lookaside_free_func *,
 189         uint32_t, size_t, uint32_t, uint16_t);
 190 static void ExDeletePagedLookasideList(paged_lookaside_list *);
 191 static void ExInitializeNPagedLookasideList(npaged_lookaside_list *,
 192         lookaside_alloc_func *, lookaside_free_func *,
 193         uint32_t, size_t, uint32_t, uint16_t);
 194 static void ExDeleteNPagedLookasideList(npaged_lookaside_list *);
 195 static slist_entry
 196         *ExInterlockedPushEntrySList(slist_header *,
 197         slist_entry *, kspin_lock *);
 198 static slist_entry
 199         *ExInterlockedPopEntrySList(slist_header *, kspin_lock *);
 200 static uint32_t InterlockedIncrement(volatile uint32_t *);
 201 static uint32_t InterlockedDecrement(volatile uint32_t *);
 202 static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t);
 203 static void *MmAllocateContiguousMemory(uint32_t, uint64_t);
 204 static void *MmAllocateContiguousMemorySpecifyCache(uint32_t,
 205         uint64_t, uint64_t, uint64_t, enum nt_caching_type);
 206 static void MmFreeContiguousMemory(void *);
 207 static void MmFreeContiguousMemorySpecifyCache(void *, uint32_t,
 208         enum nt_caching_type);
 209 static uint32_t MmSizeOfMdl(void *, size_t);
 210 static void *MmMapLockedPages(mdl *, uint8_t);
 211 static void *MmMapLockedPagesSpecifyCache(mdl *,
 212         uint8_t, uint32_t, void *, uint32_t, uint32_t);
 213 static void MmUnmapLockedPages(void *, mdl *);
 214 static device_t ntoskrnl_finddev(device_t, uint64_t, struct resource **);
 215 static void RtlZeroMemory(void *, size_t);
 216 static void RtlSecureZeroMemory(void *, size_t);
 217 static void RtlFillMemory(void *, size_t, uint8_t);
 218 static void RtlMoveMemory(void *, const void *, size_t);
 219 static ndis_status RtlCharToInteger(const char *, uint32_t, uint32_t *);
 220 static void RtlCopyMemory(void *, const void *, size_t);
 221 static size_t RtlCompareMemory(const void *, const void *, size_t);
 222 static ndis_status RtlUnicodeStringToInteger(unicode_string *,
 223         uint32_t, uint32_t *);
 224 static int atoi (const char *);
 225 static long atol (const char *);
 226 static int rand(void);
 227 static void srand(unsigned int);
 228 static void KeQuerySystemTime(uint64_t *);
 229 static uint32_t KeTickCount(void);
 230 static uint8_t IoIsWdmVersionAvailable(uint8_t, uint8_t);
 231 static int32_t IoOpenDeviceRegistryKey(struct device_object *, uint32_t,
 232     uint32_t, void **);
 233 static void ntoskrnl_thrfunc(void *);
 234 static ndis_status PsCreateSystemThread(ndis_handle *,
 235         uint32_t, void *, ndis_handle, void *, void *, void *);
 236 static ndis_status PsTerminateSystemThread(ndis_status);
 237 static ndis_status IoGetDeviceObjectPointer(unicode_string *,
 238         uint32_t, void *, device_object *);
 239 static ndis_status IoGetDeviceProperty(device_object *, uint32_t,
 240         uint32_t, void *, uint32_t *);
 241 static void KeInitializeMutex(kmutant *, uint32_t);
 242 static uint32_t KeReleaseMutex(kmutant *, uint8_t);
 243 static uint32_t KeReadStateMutex(kmutant *);
 244 static ndis_status ObReferenceObjectByHandle(ndis_handle,
 245         uint32_t, void *, uint8_t, void **, void **);
 246 static void ObfDereferenceObject(void *);
 247 static uint32_t ZwClose(ndis_handle);
 248 static uint32_t WmiQueryTraceInformation(uint32_t, void *, uint32_t,
 249         uint32_t, void *);
 250 static uint32_t WmiTraceMessage(uint64_t, uint32_t, void *, uint16_t, ...);
 251 static uint32_t IoWMIRegistrationControl(device_object *, uint32_t);
 252 static void *ntoskrnl_memset(void *, int, size_t);
 253 static void *ntoskrnl_memmove(void *, void *, size_t);
 254 static void *ntoskrnl_memchr(void *, unsigned char, size_t);
 255 static char *ntoskrnl_strstr(char *, char *);
 256 static char *ntoskrnl_strncat(char *, char *, size_t);
 257 static int ntoskrnl_toupper(int);
 258 static int ntoskrnl_tolower(int);
 259 static funcptr ntoskrnl_findwrap(funcptr);
 260 static uint32_t DbgPrint(char *, ...);
 261 static void DbgBreakPoint(void);
 262 static void KeBugCheckEx(uint32_t, u_long, u_long, u_long, u_long);
 263 static int32_t KeDelayExecutionThread(uint8_t, uint8_t, int64_t *);
 264 static int32_t KeSetPriorityThread(struct thread *, int32_t);
 265 static void dummy(void);
 266
 267 static struct mtx ntoskrnl_dispatchlock;
 268 static struct mtx ntoskrnl_interlock;
 269 static kspin_lock ntoskrnl_cancellock;
 270 static int ntoskrnl_kth = 0;
 271 static struct nt_objref_head ntoskrnl_reflist;
 272 static uma_zone_t mdl_zone;
 273 static uma_zone_t iw_zone;
 274 static struct kdpc_queue *kq_queues;
 275 static struct kdpc_queue *wq_queues;
 276 static int wq_idx = 0;
 277
 278 int
 279 ntoskrnl_libinit()
 280 {
 281         image_patch_table       *patch;
 282         int                     error;
 283         struct proc             *p;
 284         kdpc_queue              *kq;
 285         callout_entry           *e;
 286         int                     i;
 287
 288         mtx_init(&ntoskrnl_dispatchlock,
 289             "ntoskrnl dispatch lock", MTX_NDIS_LOCK, MTX_DEF|MTX_RECURSE);
 290         mtx_init(&ntoskrnl_interlock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
 291         KeInitializeSpinLock(&ntoskrnl_cancellock);
 292         KeInitializeSpinLock(&ntoskrnl_intlock);
 293         TAILQ_INIT(&ntoskrnl_reflist);
 294
 295         InitializeListHead(&ntoskrnl_calllist);
 296         InitializeListHead(&ntoskrnl_intlist);
 297         mtx_init(&ntoskrnl_calllock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
 298
 299         kq_queues = ExAllocatePoolWithTag(NonPagedPool,
 300 #ifdef NTOSKRNL_MULTIPLE_DPCS
 301             sizeof(kdpc_queue) * mp_ncpus, 0);
 302 #else
 303             sizeof(kdpc_queue), 0);
 304 #endif
 305
 306         if (kq_queues == NULL)
 307                 return (ENOMEM);
 308
 309         wq_queues = ExAllocatePoolWithTag(NonPagedPool,
 310             sizeof(kdpc_queue) * WORKITEM_THREADS, 0);
 311
 312         if (wq_queues == NULL)
 313                 return (ENOMEM);
 314
 315 #ifdef NTOSKRNL_MULTIPLE_DPCS
 316         bzero((char *)kq_queues, sizeof(kdpc_queue) * mp_ncpus);
 317 #else
 318         bzero((char *)kq_queues, sizeof(kdpc_queue));
 319 #endif
 320         bzero((char *)wq_queues, sizeof(kdpc_queue) * WORKITEM_THREADS);
 321
 322         /*
 323          * Launch the DPC threads.
 324          */
 325
 326 #ifdef NTOSKRNL_MULTIPLE_DPCS
 327         for (i = 0; i < mp_ncpus; i++) {
 328 #else
 329         for (i = 0; i < 1; i++) {
 330 #endif
 331                 kq = kq_queues + i;
 332                 kq->kq_cpu = i;
 333                 error = kproc_create(ntoskrnl_dpc_thread, kq, &p,
 334                     RFHIGHPID, NDIS_KSTACK_PAGES, "Windows DPC %d", i);
 335                 if (error)
 336                         panic("failed to launch DPC thread");
 337         }
 338
 339         /*
 340          * Launch the workitem threads.
 341          */
 342
 343         for (i = 0; i < WORKITEM_THREADS; i++) {
 344                 kq = wq_queues + i;
 345                 error = kproc_create(ntoskrnl_workitem_thread, kq, &p,
 346                     RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Workitem %d", i);
 347                 if (error)
 348                         panic("failed to launch workitem thread");
 349         }
 350
 351         patch = ntoskrnl_functbl;
 352         while (patch->ipt_func != NULL) {
 353                 windrv_wrap((funcptr)patch->ipt_func,
 354                     (funcptr *)&patch->ipt_wrap,
 355                     patch->ipt_argcnt, patch->ipt_ftype);
 356                 patch++;
 357         }
 358
 359         for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) {
 360                 e = ExAllocatePoolWithTag(NonPagedPool,
 361                     sizeof(callout_entry), 0);
 362                 if (e == NULL)
 363                         panic("failed to allocate timeouts");
 364                 mtx_lock_spin(&ntoskrnl_calllock);
 365                 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
 366                 mtx_unlock_spin(&ntoskrnl_calllock);
 367         }
 368
 369         /*
 370          * MDLs are supposed to be variable size (they describe
 371          * buffers containing some number of pages, but we don't
 372          * know ahead of time how many pages that will be). But
 373          * always allocating them off the heap is very slow. As
 374          * a compromise, we create an MDL UMA zone big enough to
 375          * handle any buffer requiring up to 16 pages, and we
 376          * use those for any MDLs for buffers of 16 pages or less
 377          * in size. For buffers larger than that (which we assume
 378          * will be few and far between, we allocate the MDLs off
 379          * the heap.
 380          */
 381
 382         mdl_zone = uma_zcreate("Windows MDL", MDL_ZONE_SIZE,
 383             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 384
 385         iw_zone = uma_zcreate("Windows WorkItem", sizeof(io_workitem),
 386             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 387
 388         return (0);
 389 }
 390
 391 int
 392 ntoskrnl_libfini()
 393 {
 394         image_patch_table       *patch;
 395         callout_entry           *e;
 396         list_entry              *l;
 397
 398         patch = ntoskrnl_functbl;
 399         while (patch->ipt_func != NULL) {
 400                 windrv_unwrap(patch->ipt_wrap);
 401                 patch++;
 402         }
 403
 404         /* Stop the workitem queues. */
 405         ntoskrnl_destroy_workitem_threads();
 406         /* Stop the DPC queues. */
 407         ntoskrnl_destroy_dpc_threads();
 408
 409         ExFreePool(kq_queues);
 410         ExFreePool(wq_queues);
 411
 412         uma_zdestroy(mdl_zone);
 413         uma_zdestroy(iw_zone);
 414
 415         mtx_lock_spin(&ntoskrnl_calllock);
 416         while(!IsListEmpty(&ntoskrnl_calllist)) {
 417                 l = RemoveHeadList(&ntoskrnl_calllist);
 418                 e = CONTAINING_RECORD(l, callout_entry, ce_list);
 419                 mtx_unlock_spin(&ntoskrnl_calllock);
 420                 ExFreePool(e);
 421                 mtx_lock_spin(&ntoskrnl_calllock);
 422         }
 423         mtx_unlock_spin(&ntoskrnl_calllock);
 424
 425         mtx_destroy(&ntoskrnl_dispatchlock);
 426         mtx_destroy(&ntoskrnl_interlock);
 427         mtx_destroy(&ntoskrnl_calllock);
 428
 429         return (0);
 430 }
 431
 432 /*
 433  * We need to be able to reference this externally from the wrapper;
 434  * GCC only generates a local implementation of memset.
 435  */
 436 static void *
 437 ntoskrnl_memset(buf, ch, size)
 438         void                    *buf;
 439         int                     ch;
 440         size_t                  size;
 441 {
 442         return (memset(buf, ch, size));
 443 }
 444
 445 static void *
 446 ntoskrnl_memmove(dst, src, size)
 447         void                    *src;
 448         void                    *dst;
 449         size_t                  size;
 450 {
 451         bcopy(src, dst, size);
 452         return (dst);
 453 }
 454
 455 static void *
 456 ntoskrnl_memchr(void *buf, unsigned char ch, size_t len)
 457 {
 458         if (len != 0) {
 459                 unsigned char *p = buf;
 460
 461                 do {
 462                         if (*p++ == ch)
 463                                 return (p - 1);
 464                 } while (--len != 0);
 465         }
 466         return (NULL);
 467 }
 468
 469 static char *
 470 ntoskrnl_strstr(s, find)
 471         char *s, *find;
 472 {
 473         char c, sc;
 474         size_t len;
 475
 476         if ((c = *find++) != 0) {
 477                 len = strlen(find);
 478                 do {
 479                         do {
 480                                 if ((sc = *s++) == 0)
 481                                         return (NULL);
 482                         } while (sc != c);
 483                 } while (strncmp(s, find, len) != 0);
 484                 s--;
 485         }
 486         return ((char *)s);
 487 }
 488
 489 /* Taken from libc */
 490 static char *
 491 ntoskrnl_strncat(dst, src, n)
 492         char            *dst;
 493         char            *src;
 494         size_t          n;
 495 {
 496         if (n != 0) {
 497                 char *d = dst;
 498                 const char *s = src;
 499
 500                 while (*d != 0)
 501                         d++;
 502                 do {
 503                         if ((*d = *s++) == 0)
 504                                 break;
 505                         d++;
 506                 } while (--n != 0);
 507                 *d = 0;
 508         }
 509         return (dst);
 510 }
 511
 512 static int
 513 ntoskrnl_toupper(c)
 514         int                     c;
 515 {
 516         return (toupper(c));
 517 }
 518
 519 static int
 520 ntoskrnl_tolower(c)
 521         int                     c;
 522 {
 523         return (tolower(c));
 524 }
 525
 526 static uint8_t
 527 RtlEqualUnicodeString(unicode_string *str1, unicode_string *str2,
 528         uint8_t caseinsensitive)
 529 {
 530         int                     i;
 531
 532         if (str1->us_len != str2->us_len)
 533                 return (FALSE);
 534
 535         for (i = 0; i < str1->us_len; i++) {
 536                 if (caseinsensitive == TRUE) {
 537                         if (toupper((char)(str1->us_buf[i] & 0xFF)) !=
 538                             toupper((char)(str2->us_buf[i] & 0xFF)))
 539                                 return (FALSE);
 540                 } else {
 541                         if (str1->us_buf[i] != str2->us_buf[i])
 542                                 return (FALSE);
 543                 }
 544         }
 545
 546         return (TRUE);
 547 }
 548
 549 static void
 550 RtlCopyString(dst, src)
 551         ansi_string             *dst;
 552         const ansi_string       *src;
 553 {
 554         if (src != NULL && src->as_buf != NULL && dst->as_buf != NULL) {
 555                 dst->as_len = min(src->as_len, dst->as_maxlen);
 556                 memcpy(dst->as_buf, src->as_buf, dst->as_len);
 557                 if (dst->as_len < dst->as_maxlen)
 558                         dst->as_buf[dst->as_len] = 0;
 559         } else
 560                 dst->as_len = 0;
 561 }
 562
 563 static void
 564 RtlCopyUnicodeString(dest, src)
 565         unicode_string          *dest;
 566         unicode_string          *src;
 567 {
 568
 569         if (dest->us_maxlen >= src->us_len)
 570                 dest->us_len = src->us_len;
 571         else
 572                 dest->us_len = dest->us_maxlen;
 573         memcpy(dest->us_buf, src->us_buf, dest->us_len);
 574 }
 575
 576 static void
 577 ntoskrnl_ascii_to_unicode(ascii, unicode, len)
 578         char                    *ascii;
 579         uint16_t                *unicode;
 580         int                     len;
 581 {
 582         int                     i;
 583         uint16_t                *ustr;
 584
 585         ustr = unicode;
 586         for (i = 0; i < len; i++) {
 587                 *ustr = (uint16_t)ascii[i];
 588                 ustr++;
 589         }
 590 }
 591
 592 static void
 593 ntoskrnl_unicode_to_ascii(unicode, ascii, len)
 594         uint16_t                *unicode;
 595         char                    *ascii;
 596         int                     len;
 597 {
 598         int                     i;
 599         uint8_t                 *astr;
 600
 601         astr = ascii;
 602         for (i = 0; i < len / 2; i++) {
 603                 *astr = (uint8_t)unicode[i];
 604                 astr++;
 605         }
 606 }
 607
 608 uint32_t
 609 RtlUnicodeStringToAnsiString(ansi_string *dest, unicode_string *src, uint8_t allocate)
 610 {
 611         if (dest == NULL || src == NULL)
 612                 return (STATUS_INVALID_PARAMETER);
 613
 614         dest->as_len = src->us_len / 2;
 615         if (dest->as_maxlen < dest->as_len)
 616                 dest->as_len = dest->as_maxlen;
 617
 618         if (allocate == TRUE) {
 619                 dest->as_buf = ExAllocatePoolWithTag(NonPagedPool,
 620                     (src->us_len / 2) + 1, 0);
 621                 if (dest->as_buf == NULL)
 622                         return (STATUS_INSUFFICIENT_RESOURCES);
 623                 dest->as_len = dest->as_maxlen = src->us_len / 2;
 624         } else {
 625                 dest->as_len = src->us_len / 2; /* XXX */
 626                 if (dest->as_maxlen < dest->as_len)
 627                         dest->as_len = dest->as_maxlen;
 628         }
 629
 630         ntoskrnl_unicode_to_ascii(src->us_buf, dest->as_buf,
 631             dest->as_len * 2);
 632
 633         return (STATUS_SUCCESS);
 634 }
 635
 636 uint32_t
 637 RtlAnsiStringToUnicodeString(unicode_string *dest, ansi_string *src,
 638         uint8_t allocate)
 639 {
 640         if (dest == NULL || src == NULL)
 641                 return (STATUS_INVALID_PARAMETER);
 642
 643         if (allocate == TRUE) {
 644                 dest->us_buf = ExAllocatePoolWithTag(NonPagedPool,
 645                     src->as_len * 2, 0);
 646                 if (dest->us_buf == NULL)
 647                         return (STATUS_INSUFFICIENT_RESOURCES);
 648                 dest->us_len = dest->us_maxlen = strlen(src->as_buf) * 2;
 649         } else {
 650                 dest->us_len = src->as_len * 2; /* XXX */
 651                 if (dest->us_maxlen < dest->us_len)
 652                         dest->us_len = dest->us_maxlen;
 653         }
 654
 655         ntoskrnl_ascii_to_unicode(src->as_buf, dest->us_buf,
 656             dest->us_len / 2);
 657
 658         return (STATUS_SUCCESS);
 659 }
 660
 661 void *
 662 ExAllocatePoolWithTag(pooltype, len, tag)
 663         uint32_t                pooltype;
 664         size_t                  len;
 665         uint32_t                tag;
 666 {
 667         void                    *buf;
 668
 669         buf = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO);
 670         if (buf == NULL)
 671                 return (NULL);
 672
 673         return (buf);
 674 }
 675
 676 static void
 677 ExFreePoolWithTag(buf, tag)
 678         void            *buf;
 679         uint32_t        tag;
 680 {
 681         ExFreePool(buf);
 682 }
 683
 684 void
 685 ExFreePool(buf)
 686         void                    *buf;
 687 {
 688         free(buf, M_DEVBUF);
 689 }
 690
 691 uint32_t
 692 IoAllocateDriverObjectExtension(drv, clid, extlen, ext)
 693         driver_object           *drv;
 694         void                    *clid;
 695         uint32_t                extlen;
 696         void                    **ext;
 697 {
 698         custom_extension        *ce;
 699
 700         ce = ExAllocatePoolWithTag(NonPagedPool, sizeof(custom_extension)
 701             + extlen, 0);
 702
 703         if (ce == NULL)
 704                 return (STATUS_INSUFFICIENT_RESOURCES);
 705
 706         ce->ce_clid = clid;
 707         InsertTailList((&drv->dro_driverext->dre_usrext), (&ce->ce_list));
 708
 709         *ext = (void *)(ce + 1);
 710
 711         return (STATUS_SUCCESS);
 712 }
 713
 714 void *
 715 IoGetDriverObjectExtension(drv, clid)
 716         driver_object           *drv;
 717         void                    *clid;
 718 {
 719         list_entry              *e;
 720         custom_extension        *ce;
 721
 722         /*
 723          * Sanity check. Our dummy bus drivers don't have
 724          * any driver extentions.
 725          */
 726
 727         if (drv->dro_driverext == NULL)
 728                 return (NULL);
 729
 730         e = drv->dro_driverext->dre_usrext.nle_flink;
 731         while (e != &drv->dro_driverext->dre_usrext) {
 732                 ce = (custom_extension *)e;
 733                 if (ce->ce_clid == clid)
 734                         return ((void *)(ce + 1));
 735                 e = e->nle_flink;
 736         }
 737
 738         return (NULL);
 739 }
 740
 741
 742 uint32_t
 743 IoCreateDevice(driver_object *drv, uint32_t devextlen, unicode_string *devname,
 744         uint32_t devtype, uint32_t devchars, uint8_t exclusive,
 745         device_object **newdev)
 746 {
 747         device_object           *dev;
 748
 749         dev = ExAllocatePoolWithTag(NonPagedPool, sizeof(device_object), 0);
 750         if (dev == NULL)
 751                 return (STATUS_INSUFFICIENT_RESOURCES);
 752
 753         dev->do_type = devtype;
 754         dev->do_drvobj = drv;
 755         dev->do_currirp = NULL;
 756         dev->do_flags = 0;
 757
 758         if (devextlen) {
 759                 dev->do_devext = ExAllocatePoolWithTag(NonPagedPool,
 760                     devextlen, 0);
 761
 762                 if (dev->do_devext == NULL) {
 763                         ExFreePool(dev);
 764                         return (STATUS_INSUFFICIENT_RESOURCES);
 765                 }
 766
 767                 bzero(dev->do_devext, devextlen);
 768         } else
 769                 dev->do_devext = NULL;
 770
 771         dev->do_size = sizeof(device_object) + devextlen;
 772         dev->do_refcnt = 1;
 773         dev->do_attacheddev = NULL;
 774         dev->do_nextdev = NULL;
 775         dev->do_devtype = devtype;
 776         dev->do_stacksize = 1;
 777         dev->do_alignreq = 1;
 778         dev->do_characteristics = devchars;
 779         dev->do_iotimer = NULL;
 780         KeInitializeEvent(&dev->do_devlock, EVENT_TYPE_SYNC, TRUE);
 781
 782         /*
 783          * Vpd is used for disk/tape devices,
 784          * but we don't support those. (Yet.)
 785          */
 786         dev->do_vpb = NULL;
 787
 788         dev->do_devobj_ext = ExAllocatePoolWithTag(NonPagedPool,
 789             sizeof(devobj_extension), 0);
 790
 791         if (dev->do_devobj_ext == NULL) {
 792                 if (dev->do_devext != NULL)
 793                         ExFreePool(dev->do_devext);
 794                 ExFreePool(dev);
 795                 return (STATUS_INSUFFICIENT_RESOURCES);
 796         }
 797
 798         dev->do_devobj_ext->dve_type = 0;
 799         dev->do_devobj_ext->dve_size = sizeof(devobj_extension);
 800         dev->do_devobj_ext->dve_devobj = dev;
 801
 802         /*
 803          * Attach this device to the driver object's list
 804          * of devices. Note: this is not the same as attaching
 805          * the device to the device stack. The driver's AddDevice
 806          * routine must explicitly call IoAddDeviceToDeviceStack()
 807          * to do that.
 808          */
 809
 810         if (drv->dro_devobj == NULL) {
 811                 drv->dro_devobj = dev;
 812                 dev->do_nextdev = NULL;
 813         } else {
 814                 dev->do_nextdev = drv->dro_devobj;
 815                 drv->dro_devobj = dev;
 816         }
 817
 818         *newdev = dev;
 819
 820         return (STATUS_SUCCESS);
 821 }
 822
 823 void
 824 IoDeleteDevice(dev)
 825         device_object           *dev;
 826 {
 827         device_object           *prev;
 828
 829         if (dev == NULL)
 830                 return;
 831
 832         if (dev->do_devobj_ext != NULL)
 833                 ExFreePool(dev->do_devobj_ext);
 834
 835         if (dev->do_devext != NULL)
 836                 ExFreePool(dev->do_devext);
 837
 838         /* Unlink the device from the driver's device list. */
 839
 840         prev = dev->do_drvobj->dro_devobj;
 841         if (prev == dev)
 842                 dev->do_drvobj->dro_devobj = dev->do_nextdev;
 843         else {
 844                 while (prev->do_nextdev != dev)
 845                         prev = prev->do_nextdev;
 846                 prev->do_nextdev = dev->do_nextdev;
 847         }
 848
 849         ExFreePool(dev);
 850 }
 851
 852 device_object *
 853 IoGetAttachedDevice(dev)
 854         device_object           *dev;
 855 {
 856         device_object           *d;
 857
 858         if (dev == NULL)
 859                 return (NULL);
 860
 861         d = dev;
 862
 863         while (d->do_attacheddev != NULL)
 864                 d = d->do_attacheddev;
 865
 866         return (d);
 867 }
 868
 869 static irp *
 870 IoBuildSynchronousFsdRequest(func, dobj, buf, len, off, event, status)
 871         uint32_t                func;
 872         device_object           *dobj;
 873         void                    *buf;
 874         uint32_t                len;
 875         uint64_t                *off;
 876         nt_kevent               *event;
 877         io_status_block         *status;
 878 {
 879         irp                     *ip;
 880
 881         ip = IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status);
 882         if (ip == NULL)
 883                 return (NULL);
 884         ip->irp_usrevent = event;
 885
 886         return (ip);
 887 }
 888
 889 static irp *
 890 IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status)
 891         uint32_t                func;
 892         device_object           *dobj;
 893         void                    *buf;
 894         uint32_t                len;
 895         uint64_t                *off;
 896         io_status_block         *status;
 897 {
 898         irp                     *ip;
 899         io_stack_location       *sl;
 900
 901         ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
 902         if (ip == NULL)
 903                 return (NULL);
 904
 905         ip->irp_usriostat = status;
 906         ip->irp_tail.irp_overlay.irp_thread = NULL;
 907
 908         sl = IoGetNextIrpStackLocation(ip);
 909         sl->isl_major = func;
 910         sl->isl_minor = 0;
 911         sl->isl_flags = 0;
 912         sl->isl_ctl = 0;
 913         sl->isl_devobj = dobj;
 914         sl->isl_fileobj = NULL;
 915         sl->isl_completionfunc = NULL;
 916
 917         ip->irp_userbuf = buf;
 918
 919         if (dobj->do_flags & DO_BUFFERED_IO) {
 920                 ip->irp_assoc.irp_sysbuf =
 921                     ExAllocatePoolWithTag(NonPagedPool, len, 0);
 922                 if (ip->irp_assoc.irp_sysbuf == NULL) {
 923                         IoFreeIrp(ip);
 924                         return (NULL);
 925                 }
 926                 bcopy(buf, ip->irp_assoc.irp_sysbuf, len);
 927         }
 928
 929         if (dobj->do_flags & DO_DIRECT_IO) {
 930                 ip->irp_mdl = IoAllocateMdl(buf, len, FALSE, FALSE, ip);
 931                 if (ip->irp_mdl == NULL) {
 932                         if (ip->irp_assoc.irp_sysbuf != NULL)
 933                                 ExFreePool(ip->irp_assoc.irp_sysbuf);
 934                         IoFreeIrp(ip);
 935                         return (NULL);
 936                 }
 937                 ip->irp_userbuf = NULL;
 938                 ip->irp_assoc.irp_sysbuf = NULL;
 939         }
 940
 941         if (func == IRP_MJ_READ) {
 942                 sl->isl_parameters.isl_read.isl_len = len;
 943                 if (off != NULL)
 944                         sl->isl_parameters.isl_read.isl_byteoff = *off;
 945                 else
 946                         sl->isl_parameters.isl_read.isl_byteoff = 0;
 947         }
 948
 949         if (func == IRP_MJ_WRITE) {
 950                 sl->isl_parameters.isl_write.isl_len = len;
 951                 if (off != NULL)
 952                         sl->isl_parameters.isl_write.isl_byteoff = *off;
 953                 else
 954                         sl->isl_parameters.isl_write.isl_byteoff = 0;
 955         }
 956
 957         return (ip);
 958 }
 959
 960 static irp *
 961 IoBuildDeviceIoControlRequest(uint32_t iocode, device_object *dobj, void *ibuf,
 962         uint32_t ilen, void *obuf, uint32_t olen, uint8_t isinternal,
 963         nt_kevent *event, io_status_block *status)
 964 {
 965         irp                     *ip;
 966         io_stack_location       *sl;
 967         uint32_t                buflen;
 968
 969         ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
 970         if (ip == NULL)
 971                 return (NULL);
 972         ip->irp_usrevent = event;
 973         ip->irp_usriostat = status;
 974         ip->irp_tail.irp_overlay.irp_thread = NULL;
 975
 976         sl = IoGetNextIrpStackLocation(ip);
 977         sl->isl_major = isinternal == TRUE ?
 978             IRP_MJ_INTERNAL_DEVICE_CONTROL : IRP_MJ_DEVICE_CONTROL;
 979         sl->isl_minor = 0;
 980         sl->isl_flags = 0;
 981         sl->isl_ctl = 0;
 982         sl->isl_devobj = dobj;
 983         sl->isl_fileobj = NULL;
 984         sl->isl_completionfunc = NULL;
 985         sl->isl_parameters.isl_ioctl.isl_iocode = iocode;
 986         sl->isl_parameters.isl_ioctl.isl_ibuflen = ilen;
 987         sl->isl_parameters.isl_ioctl.isl_obuflen = olen;
 988
 989         switch(IO_METHOD(iocode)) {
 990         case METHOD_BUFFERED:
 991                 if (ilen > olen)
 992                         buflen = ilen;
 993                 else
 994                         buflen = olen;
 995                 if (buflen) {
 996                         ip->irp_assoc.irp_sysbuf =
 997                             ExAllocatePoolWithTag(NonPagedPool, buflen, 0);
 998                         if (ip->irp_assoc.irp_sysbuf == NULL) {
 999                                 IoFreeIrp(ip);
1000                                 return (NULL);
1001                         }
1002                 }
1003                 if (ilen && ibuf != NULL) {
1004                         bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1005                         bzero((char *)ip->irp_assoc.irp_sysbuf + ilen,
1006                             buflen - ilen);
1007                 } else
1008                         bzero(ip->irp_assoc.irp_sysbuf, ilen);
1009                 ip->irp_userbuf = obuf;
1010                 break;
1011         case METHOD_IN_DIRECT:
1012         case METHOD_OUT_DIRECT:
1013                 if (ilen && ibuf != NULL) {
1014                         ip->irp_assoc.irp_sysbuf =
1015                             ExAllocatePoolWithTag(NonPagedPool, ilen, 0);
1016                         if (ip->irp_assoc.irp_sysbuf == NULL) {
1017                                 IoFreeIrp(ip);
1018                                 return (NULL);
1019                         }
1020                         bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1021                 }
1022                 if (olen && obuf != NULL) {
1023                         ip->irp_mdl = IoAllocateMdl(obuf, olen,
1024                             FALSE, FALSE, ip);
1025                         /*
1026                          * Normally we would MmProbeAndLockPages()
1027                          * here, but we don't have to in our
1028                          * imlementation.
1029                          */
1030                 }
1031                 break;
1032         case METHOD_NEITHER:
1033                 ip->irp_userbuf = obuf;
1034                 sl->isl_parameters.isl_ioctl.isl_type3ibuf = ibuf;
1035                 break;
1036         default:
1037                 break;
1038         }
1039
1040         /*
1041          * Ideally, we should associate this IRP with the calling
1042          * thread here.
1043          */
1044
1045         return (ip);
1046 }
1047
1048 static irp *
1049 IoAllocateIrp(uint8_t stsize, uint8_t chargequota)
1050 {
1051         irp                     *i;
1052
1053         i = ExAllocatePoolWithTag(NonPagedPool, IoSizeOfIrp(stsize), 0);
1054         if (i == NULL)
1055                 return (NULL);
1056
1057         IoInitializeIrp(i, IoSizeOfIrp(stsize), stsize);
1058
1059         return (i);
1060 }
1061
1062 static irp *
1063 IoMakeAssociatedIrp(irp *ip, uint8_t stsize)
1064 {
1065         irp                     *associrp;
1066
1067         associrp = IoAllocateIrp(stsize, FALSE);
1068         if (associrp == NULL)
1069                 return (NULL);
1070
1071         mtx_lock(&ntoskrnl_dispatchlock);
1072         associrp->irp_flags |= IRP_ASSOCIATED_IRP;
1073         associrp->irp_tail.irp_overlay.irp_thread =
1074             ip->irp_tail.irp_overlay.irp_thread;
1075         associrp->irp_assoc.irp_master = ip;
1076         mtx_unlock(&ntoskrnl_dispatchlock);
1077
1078         return (associrp);
1079 }
1080
1081 static void
1082 IoFreeIrp(ip)
1083         irp                     *ip;
1084 {
1085         ExFreePool(ip);
1086 }
1087
1088 static void
1089 IoInitializeIrp(irp *io, uint16_t psize, uint8_t ssize)
1090 {
1091         bzero((char *)io, IoSizeOfIrp(ssize));
1092         io->irp_size = psize;
1093         io->irp_stackcnt = ssize;
1094         io->irp_currentstackloc = ssize;
1095         InitializeListHead(&io->irp_thlist);
1096         io->irp_tail.irp_overlay.irp_csl =
1097             (io_stack_location *)(io + 1) + ssize;
1098 }
1099
1100 static void
1101 IoReuseIrp(ip, status)
1102         irp                     *ip;
1103         uint32_t                status;
1104 {
1105         uint8_t                 allocflags;
1106
1107         allocflags = ip->irp_allocflags;
1108         IoInitializeIrp(ip, ip->irp_size, ip->irp_stackcnt);
1109         ip->irp_iostat.isb_status = status;
1110         ip->irp_allocflags = allocflags;
1111 }
1112
1113 void
1114 IoAcquireCancelSpinLock(uint8_t *irql)
1115 {
1116         KeAcquireSpinLock(&ntoskrnl_cancellock, irql);
1117 }
1118
1119 void
1120 IoReleaseCancelSpinLock(uint8_t irql)
1121 {
1122         KeReleaseSpinLock(&ntoskrnl_cancellock, irql);
1123 }
1124
1125 uint8_t
1126 IoCancelIrp(irp *ip)
1127 {
1128         cancel_func             cfunc;
1129         uint8_t                 cancelirql;
1130
1131         IoAcquireCancelSpinLock(&cancelirql);
1132         cfunc = IoSetCancelRoutine(ip, NULL);
1133         ip->irp_cancel = TRUE;
1134         if (cfunc == NULL) {
1135                 IoReleaseCancelSpinLock(cancelirql);
1136                 return (FALSE);
1137         }
1138         ip->irp_cancelirql = cancelirql;
1139         MSCALL2(cfunc, IoGetCurrentIrpStackLocation(ip)->isl_devobj, ip);
1140         return (uint8_t)IoSetCancelValue(ip, TRUE);
1141 }
1142
1143 uint32_t
1144 IofCallDriver(dobj, ip)
1145         device_object           *dobj;
1146         irp                     *ip;
1147 {
1148         driver_object           *drvobj;
1149         io_stack_location       *sl;
1150         uint32_t                status;
1151         driver_dispatch         disp;
1152
1153         drvobj = dobj->do_drvobj;
1154
1155         if (ip->irp_currentstackloc <= 0)
1156                 panic("IoCallDriver(): out of stack locations");
1157
1158         IoSetNextIrpStackLocation(ip);
1159         sl = IoGetCurrentIrpStackLocation(ip);
1160
1161         sl->isl_devobj = dobj;
1162
1163         disp = drvobj->dro_dispatch[sl->isl_major];
1164         status = MSCALL2(disp, dobj, ip);
1165
1166         return (status);
1167 }
1168
1169 void
1170 IofCompleteRequest(irp *ip, uint8_t prioboost)
1171 {
1172         uint32_t                status;
1173         device_object           *dobj;
1174         io_stack_location       *sl;
1175         completion_func         cf;
1176
1177         KASSERT(ip->irp_iostat.isb_status != STATUS_PENDING,
1178             ("incorrect IRP(%p) status (STATUS_PENDING)", ip));
1179
1180         sl = IoGetCurrentIrpStackLocation(ip);
1181         IoSkipCurrentIrpStackLocation(ip);
1182
1183         do {
1184                 if (sl->isl_ctl & SL_PENDING_RETURNED)
1185                         ip->irp_pendingreturned = TRUE;
1186
1187                 if (ip->irp_currentstackloc != (ip->irp_stackcnt + 1))
1188                         dobj = IoGetCurrentIrpStackLocation(ip)->isl_devobj;
1189                 else
1190                         dobj = NULL;
1191
1192                 if (sl->isl_completionfunc != NULL &&
1193                     ((ip->irp_iostat.isb_status == STATUS_SUCCESS &&
1194                     sl->isl_ctl & SL_INVOKE_ON_SUCCESS) ||
1195                     (ip->irp_iostat.isb_status != STATUS_SUCCESS &&
1196                     sl->isl_ctl & SL_INVOKE_ON_ERROR) ||
1197                     (ip->irp_cancel == TRUE &&
1198                     sl->isl_ctl & SL_INVOKE_ON_CANCEL))) {
1199                         cf = sl->isl_completionfunc;
1200                         status = MSCALL3(cf, dobj, ip, sl->isl_completionctx);
1201                         if (status == STATUS_MORE_PROCESSING_REQUIRED)
1202                                 return;
1203                 } else {
1204                         if ((ip->irp_currentstackloc <= ip->irp_stackcnt) &&
1205                             (ip->irp_pendingreturned == TRUE))
1206                                 IoMarkIrpPending(ip);
1207                 }
1208
1209                 /* move to the next.  */
1210                 IoSkipCurrentIrpStackLocation(ip);
1211                 sl++;
1212         } while (ip->irp_currentstackloc <= (ip->irp_stackcnt + 1));
1213
1214         if (ip->irp_usriostat != NULL)
1215                 *ip->irp_usriostat = ip->irp_iostat;
1216         if (ip->irp_usrevent != NULL)
1217                 KeSetEvent(ip->irp_usrevent, prioboost, FALSE);
1218
1219         /* Handle any associated IRPs. */
1220
1221         if (ip->irp_flags & IRP_ASSOCIATED_IRP) {
1222                 uint32_t                masterirpcnt;
1223                 irp                     *masterirp;
1224                 mdl                     *m;
1225
1226                 masterirp = ip->irp_assoc.irp_master;
1227                 masterirpcnt =
1228                     InterlockedDecrement(&masterirp->irp_assoc.irp_irpcnt);
1229
1230                 while ((m = ip->irp_mdl) != NULL) {
1231                         ip->irp_mdl = m->mdl_next;
1232                         IoFreeMdl(m);
1233                 }
1234                 IoFreeIrp(ip);
1235                 if (masterirpcnt == 0)
1236                         IoCompleteRequest(masterirp, IO_NO_INCREMENT);
1237                 return;
1238         }
1239
1240         /* With any luck, these conditions will never arise. */
1241
1242         if (ip->irp_flags & IRP_PAGING_IO) {
1243                 if (ip->irp_mdl != NULL)
1244                         IoFreeMdl(ip->irp_mdl);
1245                 IoFreeIrp(ip);
1246         }
1247 }
1248
1249 void
1250 ntoskrnl_intr(arg)
1251         void                    *arg;
1252 {
1253         kinterrupt              *iobj;
1254         uint8_t                 irql;
1255         uint8_t                 claimed;
1256         list_entry              *l;
1257
1258         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1259         l = ntoskrnl_intlist.nle_flink;
1260         while (l != &ntoskrnl_intlist) {
1261                 iobj = CONTAINING_RECORD(l, kinterrupt, ki_list);
1262                 claimed = MSCALL2(iobj->ki_svcfunc, iobj, iobj->ki_svcctx);
1263                 if (claimed == TRUE)
1264                         break;
1265                 l = l->nle_flink;
1266         }
1267         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1268 }
1269
1270 uint8_t
1271 KeAcquireInterruptSpinLock(iobj)
1272         kinterrupt              *iobj;
1273 {
1274         uint8_t                 irql;
1275         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1276         return (irql);
1277 }
1278
1279 void
1280 KeReleaseInterruptSpinLock(kinterrupt *iobj, uint8_t irql)
1281 {
1282         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1283 }
1284
1285 uint8_t
1286 KeSynchronizeExecution(iobj, syncfunc, syncctx)
1287         kinterrupt              *iobj;
1288         void                    *syncfunc;
1289         void                    *syncctx;
1290 {
1291         uint8_t                 irql;
1292
1293         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1294         MSCALL1(syncfunc, syncctx);
1295         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1296
1297         return (TRUE);
1298 }
1299
1300 /*
1301  * IoConnectInterrupt() is passed only the interrupt vector and
1302  * irql that a device wants to use, but no device-specific tag
1303  * of any kind. This conflicts rather badly with FreeBSD's
1304  * bus_setup_intr(), which needs the device_t for the device
1305  * requesting interrupt delivery. In order to bypass this
1306  * inconsistency, we implement a second level of interrupt
1307  * dispatching on top of bus_setup_intr(). All devices use
1308  * ntoskrnl_intr() as their ISR, and any device requesting
1309  * interrupts will be registered with ntoskrnl_intr()'s interrupt
1310  * dispatch list. When an interrupt arrives, we walk the list
1311  * and invoke all the registered ISRs. This effectively makes all
1312  * interrupts shared, but it's the only way to duplicate the
1313  * semantics of IoConnectInterrupt() and IoDisconnectInterrupt() properly.
1314  */
1315
1316 uint32_t
1317 IoConnectInterrupt(kinterrupt **iobj, void *svcfunc, void *svcctx,
1318         kspin_lock *lock, uint32_t vector, uint8_t irql, uint8_t syncirql,
1319         uint8_t imode, uint8_t shared, uint32_t affinity, uint8_t savefloat)
1320 {
1321         uint8_t                 curirql;
1322
1323         *iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0);
1324         if (*iobj == NULL)
1325                 return (STATUS_INSUFFICIENT_RESOURCES);
1326
1327         (*iobj)->ki_svcfunc = svcfunc;
1328         (*iobj)->ki_svcctx = svcctx;
1329
1330         if (lock == NULL) {
1331                 KeInitializeSpinLock(&(*iobj)->ki_lock_priv);
1332                 (*iobj)->ki_lock = &(*iobj)->ki_lock_priv;
1333         } else
1334                 (*iobj)->ki_lock = lock;
1335
1336         KeAcquireSpinLock(&ntoskrnl_intlock, &curirql);
1337         InsertHeadList((&ntoskrnl_intlist), (&(*iobj)->ki_list));
1338         KeReleaseSpinLock(&ntoskrnl_intlock, curirql);
1339
1340         return (STATUS_SUCCESS);
1341 }
1342
1343 void
1344 IoDisconnectInterrupt(iobj)
1345         kinterrupt              *iobj;
1346 {
1347         uint8_t                 irql;
1348
1349         if (iobj == NULL)
1350                 return;
1351
1352         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1353         RemoveEntryList((&iobj->ki_list));
1354         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1355
1356         ExFreePool(iobj);
1357 }
1358
1359 device_object *
1360 IoAttachDeviceToDeviceStack(src, dst)
1361         device_object           *src;
1362         device_object           *dst;
1363 {
1364         device_object           *attached;
1365
1366         mtx_lock(&ntoskrnl_dispatchlock);
1367         attached = IoGetAttachedDevice(dst);
1368         attached->do_attacheddev = src;
1369         src->do_attacheddev = NULL;
1370         src->do_stacksize = attached->do_stacksize + 1;
1371         mtx_unlock(&ntoskrnl_dispatchlock);
1372
1373         return (attached);
1374 }
1375
1376 void
1377 IoDetachDevice(topdev)
1378         device_object           *topdev;
1379 {
1380         device_object           *tail;
1381
1382         mtx_lock(&ntoskrnl_dispatchlock);
1383
1384         /* First, break the chain. */
1385         tail = topdev->do_attacheddev;
1386         if (tail == NULL) {
1387                 mtx_unlock(&ntoskrnl_dispatchlock);
1388                 return;
1389         }
1390         topdev->do_attacheddev = tail->do_attacheddev;
1391         topdev->do_refcnt--;
1392
1393         /* Now reduce the stacksize count for the takm_il objects. */
1394
1395         tail = topdev->do_attacheddev;
1396         while (tail != NULL) {
1397                 tail->do_stacksize--;
1398                 tail = tail->do_attacheddev;
1399         }
1400
1401         mtx_unlock(&ntoskrnl_dispatchlock);
1402 }
1403
1404 /*
1405  * For the most part, an object is considered signalled if
1406  * dh_sigstate == TRUE. The exception is for mutant objects
1407  * (mutexes), where the logic works like this:
1408  *
1409  * - If the thread already owns the object and sigstate is
1410  *   less than or equal to 0, then the object is considered
1411  *   signalled (recursive acquisition).
1412  * - If dh_sigstate == 1, the object is also considered
1413  *   signalled.
1414  */
1415
1416 static int
1417 ntoskrnl_is_signalled(obj, td)
1418         nt_dispatch_header      *obj;
1419         struct thread           *td;
1420 {
1421         kmutant                 *km;
1422
1423         if (obj->dh_type == DISP_TYPE_MUTANT) {
1424                 km = (kmutant *)obj;
1425                 if ((obj->dh_sigstate <= 0 && km->km_ownerthread == td) ||
1426                     obj->dh_sigstate == 1)
1427                         return (TRUE);
1428                 return (FALSE);
1429         }
1430
1431         if (obj->dh_sigstate > 0)
1432                 return (TRUE);
1433         return (FALSE);
1434 }
1435
1436 static void
1437 ntoskrnl_satisfy_wait(obj, td)
1438         nt_dispatch_header      *obj;
1439         struct thread           *td;
1440 {
1441         kmutant                 *km;
1442
1443         switch (obj->dh_type) {
1444         case DISP_TYPE_MUTANT:
1445                 km = (struct kmutant *)obj;
1446                 obj->dh_sigstate--;
1447                 /*
1448                  * If sigstate reaches 0, the mutex is now
1449                  * non-signalled (the new thread owns it).
1450                  */
1451                 if (obj->dh_sigstate == 0) {
1452                         km->km_ownerthread = td;
1453                         if (km->km_abandoned == TRUE)
1454                                 km->km_abandoned = FALSE;
1455                 }
1456                 break;
1457         /* Synchronization objects get reset to unsignalled. */
1458         case DISP_TYPE_SYNCHRONIZATION_EVENT:
1459         case DISP_TYPE_SYNCHRONIZATION_TIMER:
1460                 obj->dh_sigstate = 0;
1461                 break;
1462         case DISP_TYPE_SEMAPHORE:
1463                 obj->dh_sigstate--;
1464                 break;
1465         default:
1466                 break;
1467         }
1468 }
1469
1470 static void
1471 ntoskrnl_satisfy_multiple_waits(wb)
1472         wait_block              *wb;
1473 {
1474         wait_block              *cur;
1475         struct thread           *td;
1476
1477         cur = wb;
1478         td = wb->wb_kthread;
1479
1480         do {
1481                 ntoskrnl_satisfy_wait(wb->wb_object, td);
1482                 cur->wb_awakened = TRUE;
1483                 cur = cur->wb_next;
1484         } while (cur != wb);
1485 }
1486
1487 /* Always called with dispatcher lock held. */
1488 static void
1489 ntoskrnl_waittest(obj, increment)
1490         nt_dispatch_header      *obj;
1491         uint32_t                increment;
1492 {
1493         wait_block              *w, *next;
1494         list_entry              *e;
1495         struct thread           *td;
1496         wb_ext                  *we;
1497         int                     satisfied;
1498
1499         /*
1500          * Once an object has been signalled, we walk its list of
1501          * wait blocks. If a wait block can be awakened, then satisfy
1502          * waits as necessary and wake the thread.
1503          *
1504          * The rules work like this:
1505          *
1506          * If a wait block is marked as WAITTYPE_ANY, then
1507          * we can satisfy the wait conditions on the current
1508          * object and wake the thread right away. Satisfying
1509          * the wait also has the effect of breaking us out
1510          * of the search loop.
1511          *
1512          * If the object is marked as WAITTYLE_ALL, then the
1513          * wait block will be part of a circularly linked
1514          * list of wait blocks belonging to a waiting thread
1515          * that's sleeping in KeWaitForMultipleObjects(). In
1516          * order to wake the thread, all the objects in the
1517          * wait list must be in the signalled state. If they
1518          * are, we then satisfy all of them and wake the
1519          * thread.
1520          *
1521          */
1522
1523         e = obj->dh_waitlisthead.nle_flink;
1524
1525         while (e != &obj->dh_waitlisthead && obj->dh_sigstate > 0) {
1526                 w = CONTAINING_RECORD(e, wait_block, wb_waitlist);
1527                 we = w->wb_ext;
1528                 td = we->we_td;
1529                 satisfied = FALSE;
1530                 if (w->wb_waittype == WAITTYPE_ANY) {
1531                         /*
1532                          * Thread can be awakened if
1533                          * any wait is satisfied.
1534                          */
1535                         ntoskrnl_satisfy_wait(obj, td);
1536                         satisfied = TRUE;
1537                         w->wb_awakened = TRUE;
1538                 } else {
1539                         /*
1540                          * Thread can only be woken up
1541                          * if all waits are satisfied.
1542                          * If the thread is waiting on multiple
1543                          * objects, they should all be linked
1544                          * through the wb_next pointers in the
1545                          * wait blocks.
1546                          */
1547                         satisfied = TRUE;
1548                         next = w->wb_next;
1549                         while (next != w) {
1550                                 if (ntoskrnl_is_signalled(obj, td) == FALSE) {
1551                                         satisfied = FALSE;
1552                                         break;
1553                                 }
1554                                 next = next->wb_next;
1555                         }
1556                         ntoskrnl_satisfy_multiple_waits(w);
1557                 }
1558
1559                 if (satisfied == TRUE)
1560                         cv_broadcastpri(&we->we_cv,
1561                             (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
1562                             w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
1563
1564                 e = e->nle_flink;
1565         }
1566 }
1567
1568 /*
1569  * Return the number of 100 nanosecond intervals since
1570  * January 1, 1601. (?!?!)
1571  */
1572 void
1573 ntoskrnl_time(tval)
1574         uint64_t                *tval;
1575 {
1576         struct timespec         ts;
1577
1578         nanotime(&ts);
1579         *tval = (uint64_t)ts.tv_nsec / 100 + (uint64_t)ts.tv_sec * 10000000 +
1580             11644473600 * 10000000; /* 100ns ticks from 1601 to 1970 */
1581 }
1582
1583 static void
1584 KeQuerySystemTime(current_time)
1585         uint64_t                *current_time;
1586 {
1587         ntoskrnl_time(current_time);
1588 }
1589
1590 static uint32_t
1591 KeTickCount(void)
1592 {
1593         struct timeval tv;
1594         getmicrouptime(&tv);
1595         return tvtohz(&tv);
1596 }
1597
1598
1599 /*
1600  * KeWaitForSingleObject() is a tricky beast, because it can be used
1601  * with several different object types: semaphores, timers, events,
1602  * mutexes and threads. Semaphores don't appear very often, but the
1603  * other object types are quite common. KeWaitForSingleObject() is
1604  * what's normally used to acquire a mutex, and it can be used to
1605  * wait for a thread termination.
1606  *
1607  * The Windows NDIS API is implemented in terms of Windows kernel
1608  * primitives, and some of the object manipulation is duplicated in
1609  * NDIS. For example, NDIS has timers and events, which are actually
1610  * Windows kevents and ktimers. Now, you're supposed to only use the
1611  * NDIS variants of these objects within the confines of the NDIS API,
1612  * but there are some naughty developers out there who will use
1613  * KeWaitForSingleObject() on NDIS timer and event objects, so we
1614  * have to support that as well. Conseqently, our NDIS timer and event
1615  * code has to be closely tied into our ntoskrnl timer and event code,
1616  * just as it is in Windows.
1617  *
1618  * KeWaitForSingleObject() may do different things for different kinds
1619  * of objects:
1620  *
1621  * - For events, we check if the event has been signalled. If the
1622  *   event is already in the signalled state, we just return immediately,
1623  *   otherwise we wait for it to be set to the signalled state by someone
1624  *   else calling KeSetEvent(). Events can be either synchronization or
1625  *   notification events.
1626  *
1627  * - For timers, if the timer has already fired and the timer is in
1628  *   the signalled state, we just return, otherwise we wait on the
1629  *   timer. Unlike an event, timers get signalled automatically when
1630  *   they expire rather than someone having to trip them manually.
1631  *   Timers initialized with KeInitializeTimer() are always notification
1632  *   events: KeInitializeTimerEx() lets you initialize a timer as
1633  *   either a notification or synchronization event.
1634  *
1635  * - For mutexes, we try to acquire the mutex and if we can't, we wait
1636  *   on the mutex until it's available and then grab it. When a mutex is
1637  *   released, it enters the signalled state, which wakes up one of the
1638  *   threads waiting to acquire it. Mutexes are always synchronization
1639  *   events.
1640  *
1641  * - For threads, the only thing we do is wait until the thread object
1642  *   enters a signalled state, which occurs when the thread terminates.
1643  *   Threads are always notification events.
1644  *
1645  * A notification event wakes up all threads waiting on an object. A
1646  * synchronization event wakes up just one. Also, a synchronization event
1647  * is auto-clearing, which means we automatically set the event back to
1648  * the non-signalled state once the wakeup is done.
1649  */
1650
1651 uint32_t
1652 KeWaitForSingleObject(void *arg, uint32_t reason, uint32_t mode,
1653     uint8_t alertable, int64_t *duetime)
1654 {
1655         wait_block              w;
1656         struct thread           *td = curthread;
1657         struct timeval          tv;
1658         int                     error = 0;
1659         uint64_t                curtime;
1660         wb_ext                  we;
1661         nt_dispatch_header      *obj;
1662
1663         obj = arg;
1664
1665         if (obj == NULL)
1666                 return (STATUS_INVALID_PARAMETER);
1667
1668         mtx_lock(&ntoskrnl_dispatchlock);
1669
1670         cv_init(&we.we_cv, "KeWFS");
1671         we.we_td = td;
1672
1673         /*
1674          * Check to see if this object is already signalled,
1675          * and just return without waiting if it is.
1676          */
1677         if (ntoskrnl_is_signalled(obj, td) == TRUE) {
1678                 /* Sanity check the signal state value. */
1679                 if (obj->dh_sigstate != INT32_MIN) {
1680                         ntoskrnl_satisfy_wait(obj, curthread);
1681                         mtx_unlock(&ntoskrnl_dispatchlock);
1682                         return (STATUS_SUCCESS);
1683                 } else {
1684                         /*
1685                          * There's a limit to how many times we can
1686                          * recursively acquire a mutant. If we hit
1687                          * the limit, something is very wrong.
1688                          */
1689                         if (obj->dh_type == DISP_TYPE_MUTANT) {
1690                                 mtx_unlock(&ntoskrnl_dispatchlock);
1691                                 panic("mutant limit exceeded");
1692                         }
1693                 }
1694         }
1695
1696         bzero((char *)&w, sizeof(wait_block));
1697         w.wb_object = obj;
1698         w.wb_ext = &we;
1699         w.wb_waittype = WAITTYPE_ANY;
1700         w.wb_next = &w;
1701         w.wb_waitkey = 0;
1702         w.wb_awakened = FALSE;
1703         w.wb_oldpri = td->td_priority;
1704
1705         InsertTailList((&obj->dh_waitlisthead), (&w.wb_waitlist));
1706
1707         /*
1708          * The timeout value is specified in 100 nanosecond units
1709          * and can be a positive or negative number. If it's positive,
1710          * then the duetime is absolute, and we need to convert it
1711          * to an absolute offset relative to now in order to use it.
1712          * If it's negative, then the duetime is relative and we
1713          * just have to convert the units.
1714          */
1715
1716         if (duetime != NULL) {
1717                 if (*duetime < 0) {
1718                         tv.tv_sec = - (*duetime) / 10000000;
1719                         tv.tv_usec = (- (*duetime) / 10) -
1720                             (tv.tv_sec * 1000000);
1721                 } else {
1722                         ntoskrnl_time(&curtime);
1723                         if (*duetime < curtime)
1724                                 tv.tv_sec = tv.tv_usec = 0;
1725                         else {
1726                                 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1727                                 tv.tv_usec = ((*duetime) - curtime) / 10 -
1728                                     (tv.tv_sec * 1000000);
1729                         }
1730                 }
1731         }
1732
1733         if (duetime == NULL)
1734                 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1735         else
1736                 error = cv_timedwait(&we.we_cv,
1737                     &ntoskrnl_dispatchlock, tvtohz(&tv));
1738
1739         RemoveEntryList(&w.wb_waitlist);
1740
1741         cv_destroy(&we.we_cv);
1742
1743         /* We timed out. Leave the object alone and return status. */
1744
1745         if (error == EWOULDBLOCK) {
1746                 mtx_unlock(&ntoskrnl_dispatchlock);
1747                 return (STATUS_TIMEOUT);
1748         }
1749
1750         mtx_unlock(&ntoskrnl_dispatchlock);
1751
1752         return (STATUS_SUCCESS);
1753 /*
1754         return (KeWaitForMultipleObjects(1, &obj, WAITTYPE_ALL, reason,
1755             mode, alertable, duetime, &w));
1756 */
1757 }
1758
1759 static uint32_t
1760 KeWaitForMultipleObjects(uint32_t cnt, nt_dispatch_header *obj[], uint32_t wtype,
1761         uint32_t reason, uint32_t mode, uint8_t alertable, int64_t *duetime,
1762         wait_block *wb_array)
1763 {
1764         struct thread           *td = curthread;
1765         wait_block              *whead, *w;
1766         wait_block              _wb_array[MAX_WAIT_OBJECTS];
1767         nt_dispatch_header      *cur;
1768         struct timeval          tv;
1769         int                     i, wcnt = 0, error = 0;
1770         uint64_t                curtime;
1771         struct timespec         t1, t2;
1772         uint32_t                status = STATUS_SUCCESS;
1773         wb_ext                  we;
1774
1775         if (cnt > MAX_WAIT_OBJECTS)
1776                 return (STATUS_INVALID_PARAMETER);
1777         if (cnt > THREAD_WAIT_OBJECTS && wb_array == NULL)
1778                 return (STATUS_INVALID_PARAMETER);
1779
1780         mtx_lock(&ntoskrnl_dispatchlock);
1781
1782         cv_init(&we.we_cv, "KeWFM");
1783         we.we_td = td;
1784
1785         if (wb_array == NULL)
1786                 whead = _wb_array;
1787         else
1788                 whead = wb_array;
1789
1790         bzero((char *)whead, sizeof(wait_block) * cnt);
1791
1792         /* First pass: see if we can satisfy any waits immediately. */
1793
1794         wcnt = 0;
1795         w = whead;
1796
1797         for (i = 0; i < cnt; i++) {
1798                 InsertTailList((&obj[i]->dh_waitlisthead),
1799                     (&w->wb_waitlist));
1800                 w->wb_ext = &we;
1801                 w->wb_object = obj[i];
1802                 w->wb_waittype = wtype;
1803                 w->wb_waitkey = i;
1804                 w->wb_awakened = FALSE;
1805                 w->wb_oldpri = td->td_priority;
1806                 w->wb_next = w + 1;
1807                 w++;
1808                 wcnt++;
1809                 if (ntoskrnl_is_signalled(obj[i], td)) {
1810                         /*
1811                          * There's a limit to how many times
1812                          * we can recursively acquire a mutant.
1813                          * If we hit the limit, something
1814                          * is very wrong.
1815                          */
1816                         if (obj[i]->dh_sigstate == INT32_MIN &&
1817                             obj[i]->dh_type == DISP_TYPE_MUTANT) {
1818                                 mtx_unlock(&ntoskrnl_dispatchlock);
1819                                 panic("mutant limit exceeded");
1820                         }
1821
1822                         /*
1823                          * If this is a WAITTYPE_ANY wait, then
1824                          * satisfy the waited object and exit
1825                          * right now.
1826                          */
1827
1828                         if (wtype == WAITTYPE_ANY) {
1829                                 ntoskrnl_satisfy_wait(obj[i], td);
1830                                 status = STATUS_WAIT_0 + i;
1831                                 goto wait_done;
1832                         } else {
1833                                 w--;
1834                                 wcnt--;
1835                                 w->wb_object = NULL;
1836                                 RemoveEntryList(&w->wb_waitlist);
1837                         }
1838                 }
1839         }
1840
1841         /*
1842          * If this is a WAITTYPE_ALL wait and all objects are
1843          * already signalled, satisfy the waits and exit now.
1844          */
1845
1846         if (wtype == WAITTYPE_ALL && wcnt == 0) {
1847                 for (i = 0; i < cnt; i++)
1848                         ntoskrnl_satisfy_wait(obj[i], td);
1849                 status = STATUS_SUCCESS;
1850                 goto wait_done;
1851         }
1852
1853         /*
1854          * Create a circular waitblock list. The waitcount
1855          * must always be non-zero when we get here.
1856          */
1857
1858         (w - 1)->wb_next = whead;
1859
1860         /* Wait on any objects that aren't yet signalled. */
1861
1862         /* Calculate timeout, if any. */
1863
1864         if (duetime != NULL) {
1865                 if (*duetime < 0) {
1866                         tv.tv_sec = - (*duetime) / 10000000;
1867                         tv.tv_usec = (- (*duetime) / 10) -
1868                             (tv.tv_sec * 1000000);
1869                 } else {
1870                         ntoskrnl_time(&curtime);
1871                         if (*duetime < curtime)
1872                                 tv.tv_sec = tv.tv_usec = 0;
1873                         else {
1874                                 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1875                                 tv.tv_usec = ((*duetime) - curtime) / 10 -
1876                                     (tv.tv_sec * 1000000);
1877                         }
1878                 }
1879         }
1880
1881         while (wcnt) {
1882                 nanotime(&t1);
1883
1884                 if (duetime == NULL)
1885                         cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1886                 else
1887                         error = cv_timedwait(&we.we_cv,
1888                             &ntoskrnl_dispatchlock, tvtohz(&tv));
1889
1890                 /* Wait with timeout expired. */
1891
1892                 if (error) {
1893                         status = STATUS_TIMEOUT;
1894                         goto wait_done;
1895                 }
1896
1897                 nanotime(&t2);
1898
1899                 /* See what's been signalled. */
1900
1901                 w = whead;
1902                 do {
1903                         cur = w->wb_object;
1904                         if (ntoskrnl_is_signalled(cur, td) == TRUE ||
1905                             w->wb_awakened == TRUE) {
1906                                 /* Sanity check the signal state value. */
1907                                 if (cur->dh_sigstate == INT32_MIN &&
1908                                     cur->dh_type == DISP_TYPE_MUTANT) {
1909                                         mtx_unlock(&ntoskrnl_dispatchlock);
1910                                         panic("mutant limit exceeded");
1911                                 }
1912                                 wcnt--;
1913                                 if (wtype == WAITTYPE_ANY) {
1914                                         status = w->wb_waitkey &
1915                                             STATUS_WAIT_0;
1916                                         goto wait_done;
1917                                 }
1918                         }
1919                         w = w->wb_next;
1920                 } while (w != whead);
1921
1922                 /*
1923                  * If all objects have been signalled, or if this
1924                  * is a WAITTYPE_ANY wait and we were woke up by
1925                  * someone, we can bail.
1926                  */
1927
1928                 if (wcnt == 0) {
1929                         status = STATUS_SUCCESS;
1930                         goto wait_done;
1931                 }
1932
1933                 /*
1934                  * If this is WAITTYPE_ALL wait, and there's still
1935                  * objects that haven't been signalled, deduct the
1936                  * time that's elapsed so far from the timeout and
1937                  * wait again (or continue waiting indefinitely if
1938                  * there's no timeout).
1939                  */
1940
1941                 if (duetime != NULL) {
1942                         tv.tv_sec -= (t2.tv_sec - t1.tv_sec);
1943                         tv.tv_usec -= (t2.tv_nsec - t1.tv_nsec) / 1000;
1944                 }
1945         }
1946
1947
1948 wait_done:
1949
1950         cv_destroy(&we.we_cv);
1951
1952         for (i = 0; i < cnt; i++) {
1953                 if (whead[i].wb_object != NULL)
1954                         RemoveEntryList(&whead[i].wb_waitlist);
1955
1956         }
1957         mtx_unlock(&ntoskrnl_dispatchlock);
1958
1959         return (status);
1960 }
1961
1962 static void
1963 WRITE_REGISTER_USHORT(uint16_t *reg, uint16_t val)
1964 {
1965         bus_space_write_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1966 }
1967
1968 static uint16_t
1969 READ_REGISTER_USHORT(reg)
1970         uint16_t                *reg;
1971 {
1972         return (bus_space_read_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1973 }
1974
1975 static void
1976 WRITE_REGISTER_ULONG(reg, val)
1977         uint32_t                *reg;
1978         uint32_t                val;
1979 {
1980         bus_space_write_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1981 }
1982
1983 static uint32_t
1984 READ_REGISTER_ULONG(reg)
1985         uint32_t                *reg;
1986 {
1987         return (bus_space_read_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1988 }
1989
1990 static uint8_t
1991 READ_REGISTER_UCHAR(uint8_t *reg)
1992 {
1993         return (bus_space_read_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1994 }
1995
1996 static void
1997 WRITE_REGISTER_UCHAR(uint8_t *reg, uint8_t val)
1998 {
1999         bus_space_write_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
2000 }
2001
2002 static int64_t
2003 _allmul(a, b)
2004         int64_t                 a;
2005         int64_t                 b;
2006 {
2007         return (a * b);
2008 }
2009
2010 static int64_t
2011 _alldiv(a, b)
2012         int64_t                 a;
2013         int64_t                 b;
2014 {
2015         return (a / b);
2016 }
2017
2018 static int64_t
2019 _allrem(a, b)
2020         int64_t                 a;
2021         int64_t                 b;
2022 {
2023         return (a % b);
2024 }
2025
2026 static uint64_t
2027 _aullmul(a, b)
2028         uint64_t                a;
2029         uint64_t                b;
2030 {
2031         return (a * b);
2032 }
2033
2034 static uint64_t
2035 _aulldiv(a, b)
2036         uint64_t                a;
2037         uint64_t                b;
2038 {
2039         return (a / b);
2040 }
2041
2042 static uint64_t
2043 _aullrem(a, b)
2044         uint64_t                a;
2045         uint64_t                b;
2046 {
2047         return (a % b);
2048 }
2049
2050 static int64_t
2051 _allshl(int64_t a, uint8_t b)
2052 {
2053         return (a << b);
2054 }
2055
2056 static uint64_t
2057 _aullshl(uint64_t a, uint8_t b)
2058 {
2059         return (a << b);
2060 }
2061
2062 static int64_t
2063 _allshr(int64_t a, uint8_t b)
2064 {
2065         return (a >> b);
2066 }
2067
2068 static uint64_t
2069 _aullshr(uint64_t a, uint8_t b)
2070 {
2071         return (a >> b);
2072 }
2073
2074 static slist_entry *
2075 ntoskrnl_pushsl(head, entry)
2076         slist_header            *head;
2077         slist_entry             *entry;
2078 {
2079         slist_entry             *oldhead;
2080
2081         oldhead = head->slh_list.slh_next;
2082         entry->sl_next = head->slh_list.slh_next;
2083         head->slh_list.slh_next = entry;
2084         head->slh_list.slh_depth++;
2085         head->slh_list.slh_seq++;
2086
2087         return (oldhead);
2088 }
2089
2090 static void
2091 InitializeSListHead(head)
2092         slist_header            *head;
2093 {
2094         memset(head, 0, sizeof(*head));
2095 }
2096
2097 static slist_entry *
2098 ntoskrnl_popsl(head)
2099         slist_header            *head;
2100 {
2101         slist_entry             *first;
2102
2103         first = head->slh_list.slh_next;
2104         if (first != NULL) {
2105                 head->slh_list.slh_next = first->sl_next;
2106                 head->slh_list.slh_depth--;
2107                 head->slh_list.slh_seq++;
2108         }
2109
2110         return (first);
2111 }
2112
2113 /*
2114  * We need this to make lookaside lists work for amd64.
2115  * We pass a pointer to ExAllocatePoolWithTag() the lookaside
2116  * list structure. For amd64 to work right, this has to be a
2117  * pointer to the wrapped version of the routine, not the
2118  * original. Letting the Windows driver invoke the original
2119  * function directly will result in a convention calling
2120  * mismatch and a pretty crash. On x86, this effectively
2121  * becomes a no-op since ipt_func and ipt_wrap are the same.
2122  */
2123
2124 static funcptr
2125 ntoskrnl_findwrap(func)
2126         funcptr                 func;
2127 {
2128         image_patch_table       *patch;
2129
2130         patch = ntoskrnl_functbl;
2131         while (patch->ipt_func != NULL) {
2132                 if ((funcptr)patch->ipt_func == func)
2133                         return ((funcptr)patch->ipt_wrap);
2134                 patch++;
2135         }
2136
2137         return (NULL);
2138 }
2139
2140 static void
2141 ExInitializePagedLookasideList(paged_lookaside_list *lookaside,
2142         lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2143         uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2144 {
2145         bzero((char *)lookaside, sizeof(paged_lookaside_list));
2146
2147         if (size < sizeof(slist_entry))
2148                 lookaside->nll_l.gl_size = sizeof(slist_entry);
2149         else
2150                 lookaside->nll_l.gl_size = size;
2151         lookaside->nll_l.gl_tag = tag;
2152         if (allocfunc == NULL)
2153                 lookaside->nll_l.gl_allocfunc =
2154                     ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2155         else
2156                 lookaside->nll_l.gl_allocfunc = allocfunc;
2157
2158         if (freefunc == NULL)
2159                 lookaside->nll_l.gl_freefunc =
2160                     ntoskrnl_findwrap((funcptr)ExFreePool);
2161         else
2162                 lookaside->nll_l.gl_freefunc = freefunc;
2163
2164 #ifdef __i386__
2165         KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2166 #endif
2167
2168         lookaside->nll_l.gl_type = NonPagedPool;
2169         lookaside->nll_l.gl_depth = depth;
2170         lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2171 }
2172
2173 static void
2174 ExDeletePagedLookasideList(lookaside)
2175         paged_lookaside_list   *lookaside;
2176 {
2177         void                    *buf;
2178         void            (*freefunc)(void *);
2179
2180         freefunc = lookaside->nll_l.gl_freefunc;
2181         while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2182                 MSCALL1(freefunc, buf);
2183 }
2184
2185 static void
2186 ExInitializeNPagedLookasideList(npaged_lookaside_list *lookaside,
2187         lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2188         uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2189 {
2190         bzero((char *)lookaside, sizeof(npaged_lookaside_list));
2191
2192         if (size < sizeof(slist_entry))
2193                 lookaside->nll_l.gl_size = sizeof(slist_entry);
2194         else
2195                 lookaside->nll_l.gl_size = size;
2196         lookaside->nll_l.gl_tag = tag;
2197         if (allocfunc == NULL)
2198                 lookaside->nll_l.gl_allocfunc =
2199                     ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2200         else
2201                 lookaside->nll_l.gl_allocfunc = allocfunc;
2202
2203         if (freefunc == NULL)
2204                 lookaside->nll_l.gl_freefunc =
2205                     ntoskrnl_findwrap((funcptr)ExFreePool);
2206         else
2207                 lookaside->nll_l.gl_freefunc = freefunc;
2208
2209 #ifdef __i386__
2210         KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2211 #endif
2212
2213         lookaside->nll_l.gl_type = NonPagedPool;
2214         lookaside->nll_l.gl_depth = depth;
2215         lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2216 }
2217
2218 static void
2219 ExDeleteNPagedLookasideList(lookaside)
2220         npaged_lookaside_list   *lookaside;
2221 {
2222         void                    *buf;
2223         void            (*freefunc)(void *);
2224
2225         freefunc = lookaside->nll_l.gl_freefunc;
2226         while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2227                 MSCALL1(freefunc, buf);
2228 }
2229
2230 slist_entry *
2231 InterlockedPushEntrySList(head, entry)
2232         slist_header            *head;
2233         slist_entry             *entry;
2234 {
2235         slist_entry             *oldhead;
2236
2237         mtx_lock_spin(&ntoskrnl_interlock);
2238         oldhead = ntoskrnl_pushsl(head, entry);
2239         mtx_unlock_spin(&ntoskrnl_interlock);
2240
2241         return (oldhead);
2242 }
2243
2244 slist_entry *
2245 InterlockedPopEntrySList(head)
2246         slist_header            *head;
2247 {
2248         slist_entry             *first;
2249
2250         mtx_lock_spin(&ntoskrnl_interlock);
2251         first = ntoskrnl_popsl(head);
2252         mtx_unlock_spin(&ntoskrnl_interlock);
2253
2254         return (first);
2255 }
2256
2257 static slist_entry *
2258 ExInterlockedPushEntrySList(head, entry, lock)
2259         slist_header            *head;
2260         slist_entry             *entry;
2261         kspin_lock              *lock;
2262 {
2263         return (InterlockedPushEntrySList(head, entry));
2264 }
2265
2266 static slist_entry *
2267 ExInterlockedPopEntrySList(head, lock)
2268         slist_header            *head;
2269         kspin_lock              *lock;
2270 {
2271         return (InterlockedPopEntrySList(head));
2272 }
2273
2274 uint16_t
2275 ExQueryDepthSList(head)
2276         slist_header            *head;
2277 {
2278         uint16_t                depth;
2279
2280         mtx_lock_spin(&ntoskrnl_interlock);
2281         depth = head->slh_list.slh_depth;
2282         mtx_unlock_spin(&ntoskrnl_interlock);
2283
2284         return (depth);
2285 }
2286
2287 void
2288 KeInitializeSpinLock(lock)
2289         kspin_lock              *lock;
2290 {
2291         *lock = 0;
2292 }
2293
2294 #ifdef __i386__
2295 void
2296 KefAcquireSpinLockAtDpcLevel(lock)
2297         kspin_lock              *lock;
2298 {
2299 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2300         int                     i = 0;
2301 #endif
2302
2303         while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) {
2304                 /* sit and spin */;
2305 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2306                 i++;
2307                 if (i > 200000000)
2308                         panic("DEADLOCK!");
2309 #endif
2310         }
2311 }
2312
2313 void
2314 KefReleaseSpinLockFromDpcLevel(lock)
2315         kspin_lock              *lock;
2316 {
2317         atomic_store_rel_int((volatile u_int *)lock, 0);
2318 }
2319
2320 uint8_t
2321 KeAcquireSpinLockRaiseToDpc(kspin_lock *lock)
2322 {
2323         uint8_t                 oldirql;
2324
2325         if (KeGetCurrentIrql() > DISPATCH_LEVEL)
2326                 panic("IRQL_NOT_LESS_THAN_OR_EQUAL");
2327
2328         KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
2329         KeAcquireSpinLockAtDpcLevel(lock);
2330
2331         return (oldirql);
2332 }
2333 #else
2334 void
2335 KeAcquireSpinLockAtDpcLevel(kspin_lock *lock)
2336 {
2337         while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0)
2338                 /* sit and spin */;
2339 }
2340
2341 void
2342 KeReleaseSpinLockFromDpcLevel(kspin_lock *lock)
2343 {
2344         atomic_store_rel_int((volatile u_int *)lock, 0);
2345 }
2346 #endif /* __i386__ */
2347
2348 uintptr_t
2349 InterlockedExchange(dst, val)
2350         volatile uint32_t       *dst;
2351         uintptr_t               val;
2352 {
2353         uintptr_t               r;
2354
2355         mtx_lock_spin(&ntoskrnl_interlock);
2356         r = *dst;
2357         *dst = val;
2358         mtx_unlock_spin(&ntoskrnl_interlock);
2359
2360         return (r);
2361 }
2362
2363 static uint32_t
2364 InterlockedIncrement(addend)
2365         volatile uint32_t       *addend;
2366 {
2367         atomic_add_long((volatile u_long *)addend, 1);
2368         return (*addend);
2369 }
2370
2371 static uint32_t
2372 InterlockedDecrement(addend)
2373         volatile uint32_t       *addend;
2374 {
2375         atomic_subtract_long((volatile u_long *)addend, 1);
2376         return (*addend);
2377 }
2378
2379 static void
2380 ExInterlockedAddLargeStatistic(addend, inc)
2381         uint64_t                *addend;
2382         uint32_t                inc;
2383 {
2384         mtx_lock_spin(&ntoskrnl_interlock);
2385         *addend += inc;
2386         mtx_unlock_spin(&ntoskrnl_interlock);
2387 };
2388
2389 mdl *
2390 IoAllocateMdl(void *vaddr, uint32_t len, uint8_t secondarybuf,
2391         uint8_t chargequota, irp *iopkt)
2392 {
2393         mdl                     *m;
2394         int                     zone = 0;
2395
2396         if (MmSizeOfMdl(vaddr, len) > MDL_ZONE_SIZE)
2397                 m = ExAllocatePoolWithTag(NonPagedPool,
2398                     MmSizeOfMdl(vaddr, len), 0);
2399         else {
2400                 m = uma_zalloc(mdl_zone, M_NOWAIT | M_ZERO);
2401                 zone++;
2402         }
2403
2404         if (m == NULL)
2405                 return (NULL);
2406
2407         MmInitializeMdl(m, vaddr, len);
2408
2409         /*
2410          * MmInitializMdl() clears the flags field, so we
2411          * have to set this here. If the MDL came from the
2412          * MDL UMA zone, tag it so we can release it to
2413          * the right place later.
2414          */
2415         if (zone)
2416                 m->mdl_flags = MDL_ZONE_ALLOCED;
2417
2418         if (iopkt != NULL) {
2419                 if (secondarybuf == TRUE) {
2420                         mdl                     *last;
2421                         last = iopkt->irp_mdl;
2422                         while (last->mdl_next != NULL)
2423                                 last = last->mdl_next;
2424                         last->mdl_next = m;
2425                 } else {
2426                         if (iopkt->irp_mdl != NULL)
2427                                 panic("leaking an MDL in IoAllocateMdl()");
2428                         iopkt->irp_mdl = m;
2429                 }
2430         }
2431
2432         return (m);
2433 }
2434
2435 void
2436 IoFreeMdl(m)
2437         mdl                     *m;
2438 {
2439         if (m == NULL)
2440                 return;
2441
2442         if (m->mdl_flags & MDL_ZONE_ALLOCED)
2443                 uma_zfree(mdl_zone, m);
2444         else
2445                 ExFreePool(m);
2446 }
2447
2448 static void *
2449 MmAllocateContiguousMemory(size, highest)
2450         uint32_t                size;
2451         uint64_t                highest;
2452 {
2453         void *addr;
2454         size_t pagelength = roundup(size, PAGE_SIZE);
2455
2456         addr = ExAllocatePoolWithTag(NonPagedPool, pagelength, 0);
2457
2458         return (addr);
2459 }
2460
2461 static void *
2462 MmAllocateContiguousMemorySpecifyCache(size, lowest, highest,
2463     boundary, cachetype)
2464         uint32_t                size;
2465         uint64_t                lowest;
2466         uint64_t                highest;
2467         uint64_t                boundary;
2468         enum nt_caching_type    cachetype;
2469 {
2470         vm_memattr_t            memattr;
2471         void                    *ret;
2472
2473         switch (cachetype) {
2474         case MmNonCached:
2475                 memattr = VM_MEMATTR_UNCACHEABLE;
2476                 break;
2477         case MmWriteCombined:
2478                 memattr = VM_MEMATTR_WRITE_COMBINING;
2479                 break;
2480         case MmNonCachedUnordered:
2481                 memattr = VM_MEMATTR_UNCACHEABLE;
2482                 break;
2483         case MmCached:
2484         case MmHardwareCoherentCached:
2485         case MmUSWCCached:
2486         default:
2487                 memattr = VM_MEMATTR_DEFAULT;
2488                 break;
2489         }
2490
2491         ret = (void *)kmem_alloc_contig(kernel_map, size, M_ZERO | M_NOWAIT,
2492             lowest, highest, PAGE_SIZE, boundary, memattr);
2493         if (ret != NULL)
2494                 malloc_type_allocated(M_DEVBUF, round_page(size));
2495         return (ret);
2496 }
2497
2498 static void
2499 MmFreeContiguousMemory(base)
2500         void                    *base;
2501 {
2502         ExFreePool(base);
2503 }
2504
2505 static void
2506 MmFreeContiguousMemorySpecifyCache(base, size, cachetype)
2507         void                    *base;
2508         uint32_t                size;
2509         enum nt_caching_type    cachetype;
2510 {
2511         contigfree(base, size, M_DEVBUF);
2512 }
2513
2514 static uint32_t
2515 MmSizeOfMdl(vaddr, len)
2516         void                    *vaddr;
2517         size_t                  len;
2518 {
2519         uint32_t                l;
2520
2521         l = sizeof(struct mdl) +
2522             (sizeof(vm_offset_t *) * SPAN_PAGES(vaddr, len));
2523
2524         return (l);
2525 }
2526
2527 /*
2528  * The Microsoft documentation says this routine fills in the
2529  * page array of an MDL with the _physical_ page addresses that
2530  * comprise the buffer, but we don't really want to do that here.
2531  * Instead, we just fill in the page array with the kernel virtual
2532  * addresses of the buffers.
2533  */
2534 void
2535 MmBuildMdlForNonPagedPool(m)
2536         mdl                     *m;
2537 {
2538         vm_offset_t             *mdl_pages;
2539         int                     pagecnt, i;
2540
2541         pagecnt = SPAN_PAGES(m->mdl_byteoffset, m->mdl_bytecount);
2542
2543         if (pagecnt > (m->mdl_size - sizeof(mdl)) / sizeof(vm_offset_t *))
2544                 panic("not enough pages in MDL to describe buffer");
2545
2546         mdl_pages = MmGetMdlPfnArray(m);
2547
2548         for (i = 0; i < pagecnt; i++)
2549                 *mdl_pages = (vm_offset_t)m->mdl_startva + (i * PAGE_SIZE);
2550
2551         m->mdl_flags |= MDL_SOURCE_IS_NONPAGED_POOL;
2552         m->mdl_mappedsystemva = MmGetMdlVirtualAddress(m);
2553 }
2554
2555 static void *
2556 MmMapLockedPages(mdl *buf, uint8_t accessmode)
2557 {
2558         buf->mdl_flags |= MDL_MAPPED_TO_SYSTEM_VA;
2559         return (MmGetMdlVirtualAddress(buf));
2560 }
2561
2562 static void *
2563 MmMapLockedPagesSpecifyCache(mdl *buf, uint8_t accessmode, uint32_t cachetype,
2564         void *vaddr, uint32_t bugcheck, uint32_t prio)
2565 {
2566         return (MmMapLockedPages(buf, accessmode));
2567 }
2568
2569 static void
2570 MmUnmapLockedPages(vaddr, buf)
2571         void                    *vaddr;
2572         mdl                     *buf;
2573 {
2574         buf->mdl_flags &= ~MDL_MAPPED_TO_SYSTEM_VA;
2575 }
2576
2577 /*
2578  * This function has a problem in that it will break if you
2579  * compile this module without PAE and try to use it on a PAE
2580  * kernel. Unfortunately, there's no way around this at the
2581  * moment. It's slightly less broken that using pmap_kextract().
2582  * You'd think the virtual memory subsystem would help us out
2583  * here, but it doesn't.
2584  */
2585
2586 static uint64_t
2587 MmGetPhysicalAddress(void *base)
2588 {
2589         return (pmap_extract(kernel_map->pmap, (vm_offset_t)base));
2590 }
2591
2592 void *
2593 MmGetSystemRoutineAddress(ustr)
2594         unicode_string          *ustr;
2595 {
2596         ansi_string             astr;
2597
2598         if (RtlUnicodeStringToAnsiString(&astr, ustr, TRUE))
2599                 return (NULL);
2600         return (ndis_get_routine_address(ntoskrnl_functbl, astr.as_buf));
2601 }
2602
2603 uint8_t
2604 MmIsAddressValid(vaddr)
2605         void                    *vaddr;
2606 {
2607         if (pmap_extract(kernel_map->pmap, (vm_offset_t)vaddr))
2608                 return (TRUE);
2609
2610         return (FALSE);
2611 }
2612
2613 void *
2614 MmMapIoSpace(paddr, len, cachetype)
2615         uint64_t                paddr;
2616         uint32_t                len;
2617         uint32_t                cachetype;
2618 {
2619         devclass_t              nexus_class;
2620         device_t                *nexus_devs, devp;
2621         int                     nexus_count = 0;
2622         device_t                matching_dev = NULL;
2623         struct resource         *res;
2624         int                     i;
2625         vm_offset_t             v;
2626
2627         /* There will always be at least one nexus. */
2628
2629         nexus_class = devclass_find("nexus");
2630         devclass_get_devices(nexus_class, &nexus_devs, &nexus_count);
2631
2632         for (i = 0; i < nexus_count; i++) {
2633                 devp = nexus_devs[i];
2634                 matching_dev = ntoskrnl_finddev(devp, paddr, &res);
2635                 if (matching_dev)
2636                         break;
2637         }
2638
2639         free(nexus_devs, M_TEMP);
2640
2641         if (matching_dev == NULL)
2642                 return (NULL);
2643
2644         v = (vm_offset_t)rman_get_virtual(res);
2645         if (paddr > rman_get_start(res))
2646                 v += paddr - rman_get_start(res);
2647
2648         return ((void *)v);
2649 }
2650
2651 void
2652 MmUnmapIoSpace(vaddr, len)
2653         void                    *vaddr;
2654         size_t                  len;
2655 {
2656 }
2657
2658
2659 static device_t
2660 ntoskrnl_finddev(dev, paddr, res)
2661         device_t                dev;
2662         uint64_t                paddr;
2663         struct resource         **res;
2664 {
2665         device_t                *children = NULL;
2666         device_t                matching_dev;
2667         int                     childcnt;
2668         struct resource         *r;
2669         struct resource_list    *rl;
2670         struct resource_list_entry      *rle;
2671         uint32_t                flags;
2672         int                     i;
2673
2674         /* We only want devices that have been successfully probed. */
2675
2676         if (device_is_alive(dev) == FALSE)
2677                 return (NULL);
2678
2679         rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
2680         if (rl != NULL) {
2681                 STAILQ_FOREACH(rle, rl, link) {
2682                         r = rle->res;
2683
2684                         if (r == NULL)
2685                                 continue;
2686
2687                         flags = rman_get_flags(r);
2688
2689                         if (rle->type == SYS_RES_MEMORY &&
2690                             paddr >= rman_get_start(r) &&
2691                             paddr <= rman_get_end(r)) {
2692                                 if (!(flags & RF_ACTIVE))
2693                                         bus_activate_resource(dev,
2694                                             SYS_RES_MEMORY, 0, r);
2695                                 *res = r;
2696                                 return (dev);
2697                         }
2698                 }
2699         }
2700
2701         /*
2702          * If this device has children, do another
2703          * level of recursion to inspect them.
2704          */
2705
2706         device_get_children(dev, &children, &childcnt);
2707
2708         for (i = 0; i < childcnt; i++) {
2709                 matching_dev = ntoskrnl_finddev(children[i], paddr, res);
2710                 if (matching_dev != NULL) {
2711                         free(children, M_TEMP);
2712                         return (matching_dev);
2713                 }
2714         }
2715
2716
2717         /* Won't somebody please think of the children! */
2718
2719         if (children != NULL)
2720                 free(children, M_TEMP);
2721
2722         return (NULL);
2723 }
2724
2725 /*
2726  * Workitems are unlike DPCs, in that they run in a user-mode thread
2727  * context rather than at DISPATCH_LEVEL in kernel context. In our
2728  * case we run them in kernel context anyway.
2729  */
2730 static void
2731 ntoskrnl_workitem_thread(arg)
2732         void                    *arg;
2733 {
2734         kdpc_queue              *kq;
2735         list_entry              *l;
2736         io_workitem             *iw;
2737         uint8_t                 irql;
2738
2739         kq = arg;
2740
2741         InitializeListHead(&kq->kq_disp);
2742         kq->kq_td = curthread;
2743         kq->kq_exit = 0;
2744         KeInitializeSpinLock(&kq->kq_lock);
2745         KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
2746
2747         while (1) {
2748                 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
2749
2750                 KeAcquireSpinLock(&kq->kq_lock, &irql);
2751
2752                 if (kq->kq_exit) {
2753                         kq->kq_exit = 0;
2754                         KeReleaseSpinLock(&kq->kq_lock, irql);
2755                         break;
2756                 }
2757
2758                 while (!IsListEmpty(&kq->kq_disp)) {
2759                         l = RemoveHeadList(&kq->kq_disp);
2760                         iw = CONTAINING_RECORD(l,
2761                             io_workitem, iw_listentry);
2762                         InitializeListHead((&iw->iw_listentry));
2763                         if (iw->iw_func == NULL)
2764                                 continue;
2765                         KeReleaseSpinLock(&kq->kq_lock, irql);
2766                         MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx);
2767                         KeAcquireSpinLock(&kq->kq_lock, &irql);
2768                 }
2769
2770                 KeReleaseSpinLock(&kq->kq_lock, irql);
2771         }
2772
2773         kproc_exit(0);
2774         return; /* notreached */
2775 }
2776
2777 static ndis_status
2778 RtlCharToInteger(src, base, val)
2779         const char              *src;
2780         uint32_t                base;
2781         uint32_t                *val;
2782 {
2783         int negative = 0;
2784         uint32_t res;
2785
2786         if (!src || !val)
2787                 return (STATUS_ACCESS_VIOLATION);
2788         while (*src != '\0' && *src <= ' ')
2789                 src++;
2790         if (*src == '+')
2791                 src++;
2792         else if (*src == '-') {
2793                 src++;
2794                 negative = 1;
2795         }
2796         if (base == 0) {
2797                 base = 10;
2798                 if (*src == '0') {
2799                         src++;
2800                         if (*src == 'b') {
2801                                 base = 2;
2802                                 src++;
2803                         } else if (*src == 'o') {
2804                                 base = 8;
2805                                 src++;
2806                         } else if (*src == 'x') {
2807                                 base = 16;
2808                                 src++;
2809                         }
2810                 }
2811         } else if (!(base == 2 || base == 8 || base == 10 || base == 16))
2812                 return (STATUS_INVALID_PARAMETER);
2813
2814         for (res = 0; *src; src++) {
2815                 int v;
2816                 if (isdigit(*src))
2817                         v = *src - '0';
2818                 else if (isxdigit(*src))
2819                         v = tolower(*src) - 'a' + 10;
2820                 else
2821                         v = base;
2822                 if (v >= base)
2823                         return (STATUS_INVALID_PARAMETER);
2824                 res = res * base + v;
2825         }
2826         *val = negative ? -res : res;
2827         return (STATUS_SUCCESS);
2828 }
2829
2830 static void
2831 ntoskrnl_destroy_workitem_threads(void)
2832 {
2833         kdpc_queue              *kq;
2834         int                     i;
2835
2836         for (i = 0; i < WORKITEM_THREADS; i++) {
2837                 kq = wq_queues + i;
2838                 kq->kq_exit = 1;
2839                 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2840                 while (kq->kq_exit)
2841                         tsleep(kq->kq_td->td_proc, PWAIT, "waitiw", hz/10);
2842         }
2843 }
2844
2845 io_workitem *
2846 IoAllocateWorkItem(dobj)
2847         device_object           *dobj;
2848 {
2849         io_workitem             *iw;
2850
2851         iw = uma_zalloc(iw_zone, M_NOWAIT);
2852         if (iw == NULL)
2853                 return (NULL);
2854
2855         InitializeListHead(&iw->iw_listentry);
2856         iw->iw_dobj = dobj;
2857
2858         mtx_lock(&ntoskrnl_dispatchlock);
2859         iw->iw_idx = wq_idx;
2860         WORKIDX_INC(wq_idx);
2861         mtx_unlock(&ntoskrnl_dispatchlock);
2862
2863         return (iw);
2864 }
2865
2866 void
2867 IoFreeWorkItem(iw)
2868         io_workitem             *iw;
2869 {
2870         uma_zfree(iw_zone, iw);
2871 }
2872
2873 void
2874 IoQueueWorkItem(iw, iw_func, qtype, ctx)
2875         io_workitem             *iw;
2876         io_workitem_func        iw_func;
2877         uint32_t                qtype;
2878         void                    *ctx;
2879 {
2880         kdpc_queue              *kq;
2881         list_entry              *l;
2882         io_workitem             *cur;
2883         uint8_t                 irql;
2884
2885         kq = wq_queues + iw->iw_idx;
2886
2887         KeAcquireSpinLock(&kq->kq_lock, &irql);
2888
2889         /*
2890          * Traverse the list and make sure this workitem hasn't
2891          * already been inserted. Queuing the same workitem
2892          * twice will hose the list but good.
2893          */
2894
2895         l = kq->kq_disp.nle_flink;
2896         while (l != &kq->kq_disp) {
2897                 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2898                 if (cur == iw) {
2899                         /* Already queued -- do nothing. */
2900                         KeReleaseSpinLock(&kq->kq_lock, irql);
2901                         return;
2902                 }
2903                 l = l->nle_flink;
2904         }
2905
2906         iw->iw_func = iw_func;
2907         iw->iw_ctx = ctx;
2908
2909         InsertTailList((&kq->kq_disp), (&iw->iw_listentry));
2910         KeReleaseSpinLock(&kq->kq_lock, irql);
2911
2912         KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2913 }
2914
2915 static void
2916 ntoskrnl_workitem(dobj, arg)
2917         device_object           *dobj;
2918         void                    *arg;
2919 {
2920         io_workitem             *iw;
2921         work_queue_item         *w;
2922         work_item_func          f;
2923
2924         iw = arg;
2925         w = (work_queue_item *)dobj;
2926         f = (work_item_func)w->wqi_func;
2927         uma_zfree(iw_zone, iw);
2928         MSCALL2(f, w, w->wqi_ctx);
2929 }
2930
2931 /*
2932  * The ExQueueWorkItem() API is deprecated in Windows XP. Microsoft
2933  * warns that it's unsafe and to use IoQueueWorkItem() instead. The
2934  * problem with ExQueueWorkItem() is that it can't guard against
2935  * the condition where a driver submits a job to the work queue and
2936  * is then unloaded before the job is able to run. IoQueueWorkItem()
2937  * acquires a reference to the device's device_object via the
2938  * object manager and retains it until after the job has completed,
2939  * which prevents the driver from being unloaded before the job
2940  * runs. (We don't currently support this behavior, though hopefully
2941  * that will change once the object manager API is fleshed out a bit.)
2942  *
2943  * Having said all that, the ExQueueWorkItem() API remains, because
2944  * there are still other parts of Windows that use it, including
2945  * NDIS itself: NdisScheduleWorkItem() calls ExQueueWorkItem().
2946  * We fake up the ExQueueWorkItem() API on top of our implementation
2947  * of IoQueueWorkItem(). Workitem thread #3 is reserved exclusively
2948  * for ExQueueWorkItem() jobs, and we pass a pointer to the work
2949  * queue item (provided by the caller) in to IoAllocateWorkItem()
2950  * instead of the device_object. We need to save this pointer so
2951  * we can apply a sanity check: as with the DPC queue and other
2952  * workitem queues, we can't allow the same work queue item to
2953  * be queued twice. If it's already pending, we silently return
2954  */
2955
2956 void
2957 ExQueueWorkItem(w, qtype)
2958         work_queue_item         *w;
2959         uint32_t                qtype;
2960 {
2961         io_workitem             *iw;
2962         io_workitem_func        iwf;
2963         kdpc_queue              *kq;
2964         list_entry              *l;
2965         io_workitem             *cur;
2966         uint8_t                 irql;
2967
2968
2969         /*
2970          * We need to do a special sanity test to make sure
2971          * the ExQueueWorkItem() API isn't used to queue
2972          * the same workitem twice. Rather than checking the
2973          * io_workitem pointer itself, we test the attached
2974          * device object, which is really a pointer to the
2975          * legacy work queue item structure.
2976          */
2977
2978         kq = wq_queues + WORKITEM_LEGACY_THREAD;
2979         KeAcquireSpinLock(&kq->kq_lock, &irql);
2980         l = kq->kq_disp.nle_flink;
2981         while (l != &kq->kq_disp) {
2982                 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2983                 if (cur->iw_dobj == (device_object *)w) {
2984                         /* Already queued -- do nothing. */
2985                         KeReleaseSpinLock(&kq->kq_lock, irql);
2986                         return;
2987                 }
2988                 l = l->nle_flink;
2989         }
2990         KeReleaseSpinLock(&kq->kq_lock, irql);
2991
2992         iw = IoAllocateWorkItem((device_object *)w);
2993         if (iw == NULL)
2994                 return;
2995
2996         iw->iw_idx = WORKITEM_LEGACY_THREAD;
2997         iwf = (io_workitem_func)ntoskrnl_findwrap((funcptr)ntoskrnl_workitem);
2998         IoQueueWorkItem(iw, iwf, qtype, iw);
2999 }
3000
3001 static void
3002 RtlZeroMemory(dst, len)
3003         void                    *dst;
3004         size_t                  len;
3005 {
3006         bzero(dst, len);
3007 }
3008
3009 static void
3010 RtlSecureZeroMemory(dst, len)
3011         void                    *dst;
3012         size_t                  len;
3013 {
3014         memset(dst, 0, len);
3015 }
3016
3017 static void
3018 RtlFillMemory(dst, len, c)
3019         void                    *dst;
3020         size_t                  len;
3021         uint8_t                 c;
3022 {
3023         memset(dst, c, len);
3024 }
3025
3026 static void
3027 RtlMoveMemory(dst, src, len)
3028         void                    *dst;
3029         const void              *src;
3030         size_t                  len;
3031 {
3032         memmove(dst, src, len);
3033 }
3034
3035 static void
3036 RtlCopyMemory(dst, src, len)
3037         void                    *dst;
3038         const void              *src;
3039         size_t                  len;
3040 {
3041         bcopy(src, dst, len);
3042 }
3043
3044 static size_t
3045 RtlCompareMemory(s1, s2, len)
3046         const void              *s1;
3047         const void              *s2;
3048         size_t                  len;
3049 {
3050         size_t                  i;
3051         uint8_t                 *m1, *m2;
3052
3053         m1 = __DECONST(char *, s1);
3054         m2 = __DECONST(char *, s2);
3055
3056         for (i = 0; i < len && m1[i] == m2[i]; i++);
3057         return (i);
3058 }
3059
3060 void
3061 RtlInitAnsiString(dst, src)
3062         ansi_string             *dst;
3063         char                    *src;
3064 {
3065         ansi_string             *a;
3066
3067         a = dst;
3068         if (a == NULL)
3069                 return;
3070         if (src == NULL) {
3071                 a->as_len = a->as_maxlen = 0;
3072                 a->as_buf = NULL;
3073         } else {
3074                 a->as_buf = src;
3075                 a->as_len = a->as_maxlen = strlen(src);
3076         }
3077 }
3078
3079 void
3080 RtlInitUnicodeString(dst, src)
3081         unicode_string          *dst;
3082         uint16_t                *src;
3083 {
3084         unicode_string          *u;
3085         int                     i;
3086
3087         u = dst;
3088         if (u == NULL)
3089                 return;
3090         if (src == NULL) {
3091                 u->us_len = u->us_maxlen = 0;
3092                 u->us_buf = NULL;
3093         } else {
3094                 i = 0;
3095                 while(src[i] != 0)
3096                         i++;
3097                 u->us_buf = src;
3098                 u->us_len = u->us_maxlen = i * 2;
3099         }
3100 }
3101
3102 ndis_status
3103 RtlUnicodeStringToInteger(ustr, base, val)
3104         unicode_string          *ustr;
3105         uint32_t                base;
3106         uint32_t                *val;
3107 {
3108         uint16_t                *uchr;
3109         int                     len, neg = 0;
3110         char                    abuf[64];
3111         char                    *astr;
3112
3113         uchr = ustr->us_buf;
3114         len = ustr->us_len;
3115         bzero(abuf, sizeof(abuf));
3116
3117         if ((char)((*uchr) & 0xFF) == '-') {
3118                 neg = 1;
3119                 uchr++;
3120                 len -= 2;
3121         } else if ((char)((*uchr) & 0xFF) == '+') {
3122                 neg = 0;
3123                 uchr++;
3124                 len -= 2;
3125         }
3126
3127         if (base == 0) {
3128                 if ((char)((*uchr) & 0xFF) == 'b') {
3129                         base = 2;
3130                         uchr++;
3131                         len -= 2;
3132                 } else if ((char)((*uchr) & 0xFF) == 'o') {
3133                         base = 8;
3134                         uchr++;
3135                         len -= 2;
3136                 } else if ((char)((*uchr) & 0xFF) == 'x') {
3137                         base = 16;
3138                         uchr++;
3139                         len -= 2;
3140                 } else
3141                         base = 10;
3142         }
3143
3144         astr = abuf;
3145         if (neg) {
3146                 strcpy(astr, "-");
3147                 astr++;
3148         }
3149
3150         ntoskrnl_unicode_to_ascii(uchr, astr, len);
3151         *val = strtoul(abuf, NULL, base);
3152
3153         return (STATUS_SUCCESS);
3154 }
3155
3156 void
3157 RtlFreeUnicodeString(ustr)
3158         unicode_string          *ustr;
3159 {
3160         if (ustr->us_buf == NULL)
3161                 return;
3162         ExFreePool(ustr->us_buf);
3163         ustr->us_buf = NULL;
3164 }
3165
3166 void
3167 RtlFreeAnsiString(astr)
3168         ansi_string             *astr;
3169 {
3170         if (astr->as_buf == NULL)
3171                 return;
3172         ExFreePool(astr->as_buf);
3173         astr->as_buf = NULL;
3174 }
3175
3176 static int
3177 atoi(str)
3178         const char              *str;
3179 {
3180         return (int)strtol(str, (char **)NULL, 10);
3181 }
3182
3183 static long
3184 atol(str)
3185         const char              *str;
3186 {
3187         return strtol(str, (char **)NULL, 10);
3188 }
3189
3190 static int
3191 rand(void)
3192 {
3193         struct timeval          tv;
3194
3195         microtime(&tv);
3196         srandom(tv.tv_usec);
3197         return ((int)random());
3198 }
3199
3200 static void
3201 srand(seed)
3202         unsigned int            seed;
3203 {
3204         srandom(seed);
3205 }
3206
3207 static uint8_t
3208 IoIsWdmVersionAvailable(uint8_t major, uint8_t minor)
3209 {
3210         if (major == WDM_MAJOR && minor == WDM_MINOR_WINXP)
3211                 return (TRUE);
3212         return (FALSE);
3213 }
3214
3215 static int32_t
3216 IoOpenDeviceRegistryKey(struct device_object *devobj, uint32_t type,
3217     uint32_t mask, void **key)
3218 {
3219         return (NDIS_STATUS_INVALID_DEVICE_REQUEST);
3220 }
3221
3222 static ndis_status
3223 IoGetDeviceObjectPointer(name, reqaccess, fileobj, devobj)
3224         unicode_string          *name;
3225         uint32_t                reqaccess;
3226         void                    *fileobj;
3227         device_object           *devobj;
3228 {
3229         return (STATUS_SUCCESS);
3230 }
3231
3232 static ndis_status
3233 IoGetDeviceProperty(devobj, regprop, buflen, prop, reslen)
3234         device_object           *devobj;
3235         uint32_t                regprop;
3236         uint32_t                buflen;
3237         void                    *prop;
3238         uint32_t                *reslen;
3239 {
3240         driver_object           *drv;
3241         uint16_t                **name;
3242
3243         drv = devobj->do_drvobj;
3244
3245         switch (regprop) {
3246         case DEVPROP_DRIVER_KEYNAME:
3247                 name = prop;
3248                 *name = drv->dro_drivername.us_buf;
3249                 *reslen = drv->dro_drivername.us_len;
3250                 break;
3251         default:
3252                 return (STATUS_INVALID_PARAMETER_2);
3253                 break;
3254         }
3255
3256         return (STATUS_SUCCESS);
3257 }
3258
3259 static void
3260 KeInitializeMutex(kmutex, level)
3261         kmutant                 *kmutex;
3262         uint32_t                level;
3263 {
3264         InitializeListHead((&kmutex->km_header.dh_waitlisthead));
3265         kmutex->km_abandoned = FALSE;
3266         kmutex->km_apcdisable = 1;
3267         kmutex->km_header.dh_sigstate = 1;
3268         kmutex->km_header.dh_type = DISP_TYPE_MUTANT;
3269         kmutex->km_header.dh_size = sizeof(kmutant) / sizeof(uint32_t);
3270         kmutex->km_ownerthread = NULL;
3271 }
3272
3273 static uint32_t
3274 KeReleaseMutex(kmutant *kmutex, uint8_t kwait)
3275 {
3276         uint32_t                prevstate;
3277
3278         mtx_lock(&ntoskrnl_dispatchlock);
3279         prevstate = kmutex->km_header.dh_sigstate;
3280         if (kmutex->km_ownerthread != curthread) {
3281                 mtx_unlock(&ntoskrnl_dispatchlock);
3282                 return (STATUS_MUTANT_NOT_OWNED);
3283         }
3284
3285         kmutex->km_header.dh_sigstate++;
3286         kmutex->km_abandoned = FALSE;
3287
3288         if (kmutex->km_header.dh_sigstate == 1) {
3289                 kmutex->km_ownerthread = NULL;
3290                 ntoskrnl_waittest(&kmutex->km_header, IO_NO_INCREMENT);
3291         }
3292
3293         mtx_unlock(&ntoskrnl_dispatchlock);
3294
3295         return (prevstate);
3296 }
3297
3298 static uint32_t
3299 KeReadStateMutex(kmutex)
3300         kmutant                 *kmutex;
3301 {
3302         return (kmutex->km_header.dh_sigstate);
3303 }
3304
3305 void
3306 KeInitializeEvent(nt_kevent *kevent, uint32_t type, uint8_t state)
3307 {
3308         InitializeListHead((&kevent->k_header.dh_waitlisthead));
3309         kevent->k_header.dh_sigstate = state;
3310         if (type == EVENT_TYPE_NOTIFY)
3311                 kevent->k_header.dh_type = DISP_TYPE_NOTIFICATION_EVENT;
3312         else
3313                 kevent->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_EVENT;
3314         kevent->k_header.dh_size = sizeof(nt_kevent) / sizeof(uint32_t);
3315 }
3316
3317 uint32_t
3318 KeResetEvent(kevent)
3319         nt_kevent               *kevent;
3320 {
3321         uint32_t                prevstate;
3322
3323         mtx_lock(&ntoskrnl_dispatchlock);
3324         prevstate = kevent->k_header.dh_sigstate;
3325         kevent->k_header.dh_sigstate = FALSE;
3326         mtx_unlock(&ntoskrnl_dispatchlock);
3327
3328         return (prevstate);
3329 }
3330
3331 uint32_t
3332 KeSetEvent(nt_kevent *kevent, uint32_t increment, uint8_t kwait)
3333 {
3334         uint32_t                prevstate;
3335         wait_block              *w;
3336         nt_dispatch_header      *dh;
3337         struct thread           *td;
3338         wb_ext                  *we;
3339
3340         mtx_lock(&ntoskrnl_dispatchlock);
3341         prevstate = kevent->k_header.dh_sigstate;
3342         dh = &kevent->k_header;
3343
3344         if (IsListEmpty(&dh->dh_waitlisthead))
3345                 /*
3346                  * If there's nobody in the waitlist, just set
3347                  * the state to signalled.
3348                  */
3349                 dh->dh_sigstate = 1;
3350         else {
3351                 /*
3352                  * Get the first waiter. If this is a synchronization
3353                  * event, just wake up that one thread (don't bother
3354                  * setting the state to signalled since we're supposed
3355                  * to automatically clear synchronization events anyway).
3356                  *
3357                  * If it's a notification event, or the the first
3358                  * waiter is doing a WAITTYPE_ALL wait, go through
3359                  * the full wait satisfaction process.
3360                  */
3361                 w = CONTAINING_RECORD(dh->dh_waitlisthead.nle_flink,
3362                     wait_block, wb_waitlist);
3363                 we = w->wb_ext;
3364                 td = we->we_td;
3365                 if (kevent->k_header.dh_type == DISP_TYPE_NOTIFICATION_EVENT ||
3366                     w->wb_waittype == WAITTYPE_ALL) {
3367                         if (prevstate == 0) {
3368                                 dh->dh_sigstate = 1;
3369                                 ntoskrnl_waittest(dh, increment);
3370                         }
3371                 } else {
3372                         w->wb_awakened |= TRUE;
3373                         cv_broadcastpri(&we->we_cv,
3374                             (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
3375                             w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
3376                 }
3377         }
3378
3379         mtx_unlock(&ntoskrnl_dispatchlock);
3380
3381         return (prevstate);
3382 }
3383
3384 void
3385 KeClearEvent(kevent)
3386         nt_kevent               *kevent;
3387 {
3388         kevent->k_header.dh_sigstate = FALSE;
3389 }
3390
3391 uint32_t
3392 KeReadStateEvent(kevent)
3393         nt_kevent               *kevent;
3394 {
3395         return (kevent->k_header.dh_sigstate);
3396 }
3397
3398 /*
3399  * The object manager in Windows is responsible for managing
3400  * references and access to various types of objects, including
3401  * device_objects, events, threads, timers and so on. However,
3402  * there's a difference in the way objects are handled in user
3403  * mode versus kernel mode.
3404  *
3405  * In user mode (i.e. Win32 applications), all objects are
3406  * managed by the object manager. For example, when you create
3407  * a timer or event object, you actually end up with an
3408  * object_header (for the object manager's bookkeeping
3409  * purposes) and an object body (which contains the actual object
3410  * structure, e.g. ktimer, kevent, etc...). This allows Windows
3411  * to manage resource quotas and to enforce access restrictions
3412  * on basically every kind of system object handled by the kernel.
3413  *
3414  * However, in kernel mode, you only end up using the object
3415  * manager some of the time. For example, in a driver, you create
3416  * a timer object by simply allocating the memory for a ktimer
3417  * structure and initializing it with KeInitializeTimer(). Hence,
3418  * the timer has no object_header and no reference counting or
3419  * security/resource checks are done on it. The assumption in
3420  * this case is that if you're running in kernel mode, you know
3421  * what you're doing, and you're already at an elevated privilege
3422  * anyway.
3423  *
3424  * There are some exceptions to this. The two most important ones
3425  * for our purposes are device_objects and threads. We need to use
3426  * the object manager to do reference counting on device_objects,
3427  * and for threads, you can only get a pointer to a thread's
3428  * dispatch header by using ObReferenceObjectByHandle() on the
3429  * handle returned by PsCreateSystemThread().
3430  */
3431
3432 static ndis_status
3433 ObReferenceObjectByHandle(ndis_handle handle, uint32_t reqaccess, void *otype,
3434         uint8_t accessmode, void **object, void **handleinfo)
3435 {
3436         nt_objref               *nr;
3437
3438         nr = malloc(sizeof(nt_objref), M_DEVBUF, M_NOWAIT|M_ZERO);
3439         if (nr == NULL)
3440                 return (STATUS_INSUFFICIENT_RESOURCES);
3441
3442         InitializeListHead((&nr->no_dh.dh_waitlisthead));
3443         nr->no_obj = handle;
3444         nr->no_dh.dh_type = DISP_TYPE_THREAD;
3445         nr->no_dh.dh_sigstate = 0;
3446         nr->no_dh.dh_size = (uint8_t)(sizeof(struct thread) /
3447             sizeof(uint32_t));
3448         TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link);
3449         *object = nr;
3450
3451         return (STATUS_SUCCESS);
3452 }
3453
3454 static void
3455 ObfDereferenceObject(object)
3456         void                    *object;
3457 {
3458         nt_objref               *nr;
3459
3460         nr = object;
3461         TAILQ_REMOVE(&ntoskrnl_reflist, nr, link);
3462         free(nr, M_DEVBUF);
3463 }
3464
3465 static uint32_t
3466 ZwClose(handle)
3467         ndis_handle             handle;
3468 {
3469         return (STATUS_SUCCESS);
3470 }
3471
3472 static uint32_t
3473 WmiQueryTraceInformation(traceclass, traceinfo, infolen, reqlen, buf)
3474         uint32_t                traceclass;
3475         void                    *traceinfo;
3476         uint32_t                infolen;
3477         uint32_t                reqlen;
3478         void                    *buf;
3479 {
3480         return (STATUS_NOT_FOUND);
3481 }
3482
3483 static uint32_t
3484 WmiTraceMessage(uint64_t loghandle, uint32_t messageflags,
3485         void *guid, uint16_t messagenum, ...)
3486 {
3487         return (STATUS_SUCCESS);
3488 }
3489
3490 static uint32_t
3491 IoWMIRegistrationControl(dobj, action)
3492         device_object           *dobj;
3493         uint32_t                action;
3494 {
3495         return (STATUS_SUCCESS);
3496 }
3497
3498 /*
3499  * This is here just in case the thread returns without calling
3500  * PsTerminateSystemThread().
3501  */
3502 static void
3503 ntoskrnl_thrfunc(arg)
3504         void                    *arg;
3505 {
3506         thread_context          *thrctx;
3507         uint32_t (*tfunc)(void *);
3508         void                    *tctx;
3509         uint32_t                rval;
3510
3511         thrctx = arg;
3512         tfunc = thrctx->tc_thrfunc;
3513         tctx = thrctx->tc_thrctx;
3514         free(thrctx, M_TEMP);
3515
3516         rval = MSCALL1(tfunc, tctx);
3517
3518         PsTerminateSystemThread(rval);
3519         return; /* notreached */
3520 }
3521
3522 static ndis_status
3523 PsCreateSystemThread(handle, reqaccess, objattrs, phandle,
3524         clientid, thrfunc, thrctx)
3525         ndis_handle             *handle;
3526         uint32_t                reqaccess;
3527         void                    *objattrs;
3528         ndis_handle             phandle;
3529         void                    *clientid;
3530         void                    *thrfunc;
3531         void                    *thrctx;
3532 {
3533         int                     error;
3534         thread_context          *tc;
3535         struct proc             *p;
3536
3537         tc = malloc(sizeof(thread_context), M_TEMP, M_NOWAIT);
3538         if (tc == NULL)
3539                 return (STATUS_INSUFFICIENT_RESOURCES);
3540
3541         tc->tc_thrctx = thrctx;
3542         tc->tc_thrfunc = thrfunc;
3543
3544         error = kproc_create(ntoskrnl_thrfunc, tc, &p,
3545             RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Kthread %d", ntoskrnl_kth);
3546
3547         if (error) {
3548                 free(tc, M_TEMP);
3549                 return (STATUS_INSUFFICIENT_RESOURCES);
3550         }
3551
3552         *handle = p;
3553         ntoskrnl_kth++;
3554
3555         return (STATUS_SUCCESS);
3556 }
3557
3558 /*
3559  * In Windows, the exit of a thread is an event that you're allowed
3560  * to wait on, assuming you've obtained a reference to the thread using
3561  * ObReferenceObjectByHandle(). Unfortunately, the only way we can
3562  * simulate this behavior is to register each thread we create in a
3563  * reference list, and if someone holds a reference to us, we poke
3564  * them.
3565  */
3566 static ndis_status
3567 PsTerminateSystemThread(status)
3568         ndis_status             status;
3569 {
3570         struct nt_objref        *nr;
3571
3572         mtx_lock(&ntoskrnl_dispatchlock);
3573         TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) {
3574                 if (nr->no_obj != curthread->td_proc)
3575                         continue;
3576                 nr->no_dh.dh_sigstate = 1;
3577                 ntoskrnl_waittest(&nr->no_dh, IO_NO_INCREMENT);
3578                 break;
3579         }
3580         mtx_unlock(&ntoskrnl_dispatchlock);
3581
3582         ntoskrnl_kth--;
3583
3584         kproc_exit(0);
3585         return (0);     /* notreached */
3586 }
3587
3588 static uint32_t
3589 DbgPrint(char *fmt, ...)
3590 {
3591         va_list                 ap;
3592
3593         if (bootverbose) {
3594                 va_start(ap, fmt);
3595                 vprintf(fmt, ap);
3596         }
3597
3598         return (STATUS_SUCCESS);
3599 }
3600
3601 static void
3602 DbgBreakPoint(void)
3603 {
3604
3605         kdb_enter(KDB_WHY_NDIS, "DbgBreakPoint(): breakpoint");
3606 }
3607
3608 static void
3609 KeBugCheckEx(code, param1, param2, param3, param4)
3610     uint32_t                    code;
3611     u_long                      param1;
3612     u_long                      param2;
3613     u_long                      param3;
3614     u_long                      param4;
3615 {
3616         panic("KeBugCheckEx: STOP 0x%X", code);
3617 }
3618
3619 static void
3620 ntoskrnl_timercall(arg)
3621         void                    *arg;
3622 {
3623         ktimer                  *timer;
3624         struct timeval          tv;
3625         kdpc                    *dpc;
3626
3627         mtx_lock(&ntoskrnl_dispatchlock);
3628
3629         timer = arg;
3630
3631 #ifdef NTOSKRNL_DEBUG_TIMERS
3632         ntoskrnl_timer_fires++;
3633 #endif
3634         ntoskrnl_remove_timer(timer);
3635
3636         /*
3637          * This should never happen, but complain
3638          * if it does.
3639          */
3640
3641         if (timer->k_header.dh_inserted == FALSE) {
3642                 mtx_unlock(&ntoskrnl_dispatchlock);
3643                 printf("NTOS: timer %p fired even though "
3644                     "it was canceled\n", timer);
3645                 return;
3646         }
3647
3648         /* Mark the timer as no longer being on the timer queue. */
3649
3650         timer->k_header.dh_inserted = FALSE;
3651
3652         /* Now signal the object and satisfy any waits on it. */
3653
3654         timer->k_header.dh_sigstate = 1;
3655         ntoskrnl_waittest(&timer->k_header, IO_NO_INCREMENT);
3656
3657         /*
3658          * If this is a periodic timer, re-arm it
3659          * so it will fire again. We do this before
3660          * calling any deferred procedure calls because
3661          * it's possible the DPC might cancel the timer,
3662          * in which case it would be wrong for us to
3663          * re-arm it again afterwards.
3664          */
3665
3666         if (timer->k_period) {
3667                 tv.tv_sec = 0;
3668                 tv.tv_usec = timer->k_period * 1000;
3669                 timer->k_header.dh_inserted = TRUE;
3670                 ntoskrnl_insert_timer(timer, tvtohz(&tv));
3671 #ifdef NTOSKRNL_DEBUG_TIMERS
3672                 ntoskrnl_timer_reloads++;
3673 #endif
3674         }
3675
3676         dpc = timer->k_dpc;
3677
3678         mtx_unlock(&ntoskrnl_dispatchlock);
3679
3680         /* If there's a DPC associated with the timer, queue it up. */
3681
3682         if (dpc != NULL)
3683                 KeInsertQueueDpc(dpc, NULL, NULL);
3684 }
3685
3686 #ifdef NTOSKRNL_DEBUG_TIMERS
3687 static int
3688 sysctl_show_timers(SYSCTL_HANDLER_ARGS)
3689 {
3690         int                     ret;
3691
3692         ret = 0;
3693         ntoskrnl_show_timers();
3694         return (sysctl_handle_int(oidp, &ret, 0, req));
3695 }
3696
3697 static void
3698 ntoskrnl_show_timers()
3699 {
3700         int                     i = 0;
3701         list_entry              *l;
3702
3703         mtx_lock_spin(&ntoskrnl_calllock);
3704         l = ntoskrnl_calllist.nle_flink;
3705         while(l != &ntoskrnl_calllist) {
3706                 i++;
3707                 l = l->nle_flink;
3708         }
3709         mtx_unlock_spin(&ntoskrnl_calllock);
3710
3711         printf("\n");
3712         printf("%d timers available (out of %d)\n", i, NTOSKRNL_TIMEOUTS);
3713         printf("timer sets: %qu\n", ntoskrnl_timer_sets);
3714         printf("timer reloads: %qu\n", ntoskrnl_timer_reloads);
3715         printf("timer cancels: %qu\n", ntoskrnl_timer_cancels);
3716         printf("timer fires: %qu\n", ntoskrnl_timer_fires);
3717         printf("\n");
3718 }
3719 #endif
3720
3721 /*
3722  * Must be called with dispatcher lock held.
3723  */
3724
3725 static void
3726 ntoskrnl_insert_timer(timer, ticks)
3727         ktimer                  *timer;
3728         int                     ticks;
3729 {
3730         callout_entry           *e;
3731         list_entry              *l;
3732         struct callout          *c;
3733
3734         /*
3735          * Try and allocate a timer.
3736          */
3737         mtx_lock_spin(&ntoskrnl_calllock);
3738         if (IsListEmpty(&ntoskrnl_calllist)) {
3739                 mtx_unlock_spin(&ntoskrnl_calllock);
3740 #ifdef NTOSKRNL_DEBUG_TIMERS
3741                 ntoskrnl_show_timers();
3742 #endif
3743                 panic("out of timers!");
3744         }
3745         l = RemoveHeadList(&ntoskrnl_calllist);
3746         mtx_unlock_spin(&ntoskrnl_calllock);
3747
3748         e = CONTAINING_RECORD(l, callout_entry, ce_list);
3749         c = &e->ce_callout;
3750
3751         timer->k_callout = c;
3752
3753         callout_init(c, CALLOUT_MPSAFE);
3754         callout_reset(c, ticks, ntoskrnl_timercall, timer);
3755 }
3756
3757 static void
3758 ntoskrnl_remove_timer(timer)
3759         ktimer                  *timer;
3760 {
3761         callout_entry           *e;
3762
3763         e = (callout_entry *)timer->k_callout;
3764         callout_stop(timer->k_callout);
3765
3766         mtx_lock_spin(&ntoskrnl_calllock);
3767         InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
3768         mtx_unlock_spin(&ntoskrnl_calllock);
3769 }
3770
3771 void
3772 KeInitializeTimer(timer)
3773         ktimer                  *timer;
3774 {
3775         if (timer == NULL)
3776                 return;
3777
3778         KeInitializeTimerEx(timer,  EVENT_TYPE_NOTIFY);
3779 }
3780
3781 void
3782 KeInitializeTimerEx(timer, type)
3783         ktimer                  *timer;
3784         uint32_t                type;
3785 {
3786         if (timer == NULL)
3787                 return;
3788
3789         bzero((char *)timer, sizeof(ktimer));
3790         InitializeListHead((&timer->k_header.dh_waitlisthead));
3791         timer->k_header.dh_sigstate = FALSE;
3792         timer->k_header.dh_inserted = FALSE;
3793         if (type == EVENT_TYPE_NOTIFY)
3794                 timer->k_header.dh_type = DISP_TYPE_NOTIFICATION_TIMER;
3795         else
3796                 timer->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_TIMER;
3797         timer->k_header.dh_size = sizeof(ktimer) / sizeof(uint32_t);
3798 }
3799
3800 /*
3801  * DPC subsystem. A Windows Defered Procedure Call has the following
3802  * properties:
3803  * - It runs at DISPATCH_LEVEL.
3804  * - It can have one of 3 importance values that control when it
3805  *   runs relative to other DPCs in the queue.
3806  * - On SMP systems, it can be set to run on a specific processor.
3807  * In order to satisfy the last property, we create a DPC thread for
3808  * each CPU in the system and bind it to that CPU. Each thread
3809  * maintains three queues with different importance levels, which
3810  * will be processed in order from lowest to highest.
3811  *
3812  * In Windows, interrupt handlers run as DPCs. (Not to be confused
3813  * with ISRs, which run in interrupt context and can preempt DPCs.)
3814  * ISRs are given the highest importance so that they'll take
3815  * precedence over timers and other things.
3816  */
3817
3818 static void
3819 ntoskrnl_dpc_thread(arg)
3820         void                    *arg;
3821 {
3822         kdpc_queue              *kq;
3823         kdpc                    *d;
3824         list_entry              *l;
3825         uint8_t                 irql;
3826
3827         kq = arg;
3828
3829         InitializeListHead(&kq->kq_disp);
3830         kq->kq_td = curthread;
3831         kq->kq_exit = 0;
3832         kq->kq_running = FALSE;
3833         KeInitializeSpinLock(&kq->kq_lock);
3834         KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
3835         KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE);
3836
3837         /*
3838          * Elevate our priority. DPCs are used to run interrupt
3839          * handlers, and they should trigger as soon as possible
3840          * once scheduled by an ISR.
3841          */
3842
3843         thread_lock(curthread);
3844 #ifdef NTOSKRNL_MULTIPLE_DPCS
3845         sched_bind(curthread, kq->kq_cpu);
3846 #endif
3847         sched_prio(curthread, PRI_MIN_KERN);
3848         thread_unlock(curthread);
3849
3850         while (1) {
3851                 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
3852
3853                 KeAcquireSpinLock(&kq->kq_lock, &irql);
3854
3855                 if (kq->kq_exit) {
3856                         kq->kq_exit = 0;
3857                         KeReleaseSpinLock(&kq->kq_lock, irql);
3858                         break;
3859                 }
3860
3861                 kq->kq_running = TRUE;
3862
3863                 while (!IsListEmpty(&kq->kq_disp)) {
3864                         l = RemoveHeadList((&kq->kq_disp));
3865                         d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3866                         InitializeListHead((&d->k_dpclistentry));
3867                         KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
3868                         MSCALL4(d->k_deferedfunc, d, d->k_deferredctx,
3869                             d->k_sysarg1, d->k_sysarg2);
3870                         KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3871                 }
3872
3873                 kq->kq_running = FALSE;
3874
3875                 KeReleaseSpinLock(&kq->kq_lock, irql);
3876
3877                 KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE);
3878         }
3879
3880         kproc_exit(0);
3881         return; /* notreached */
3882 }
3883
3884 static void
3885 ntoskrnl_destroy_dpc_threads(void)
3886 {
3887         kdpc_queue              *kq;
3888         kdpc                    dpc;
3889         int                     i;
3890
3891         kq = kq_queues;
3892 #ifdef NTOSKRNL_MULTIPLE_DPCS
3893         for (i = 0; i < mp_ncpus; i++) {
3894 #else
3895         for (i = 0; i < 1; i++) {
3896 #endif
3897                 kq += i;
3898
3899                 kq->kq_exit = 1;
3900                 KeInitializeDpc(&dpc, NULL, NULL);
3901                 KeSetTargetProcessorDpc(&dpc, i);
3902                 KeInsertQueueDpc(&dpc, NULL, NULL);
3903                 while (kq->kq_exit)
3904                         tsleep(kq->kq_td->td_proc, PWAIT, "dpcw", hz/10);
3905         }
3906 }
3907
3908 static uint8_t
3909 ntoskrnl_insert_dpc(head, dpc)
3910         list_entry              *head;
3911         kdpc                    *dpc;
3912 {
3913         list_entry              *l;
3914         kdpc                    *d;
3915
3916         l = head->nle_flink;
3917         while (l != head) {
3918                 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3919                 if (d == dpc)
3920                         return (FALSE);
3921                 l = l->nle_flink;
3922         }
3923
3924         if (dpc->k_importance == KDPC_IMPORTANCE_LOW)
3925                 InsertTailList((head), (&dpc->k_dpclistentry));
3926         else
3927                 InsertHeadList((head), (&dpc->k_dpclistentry));
3928
3929         return (TRUE);
3930 }
3931
3932 void
3933 KeInitializeDpc(dpc, dpcfunc, dpcctx)
3934         kdpc                    *dpc;
3935         void                    *dpcfunc;
3936         void                    *dpcctx;
3937 {
3938
3939         if (dpc == NULL)
3940                 return;
3941
3942         dpc->k_deferedfunc = dpcfunc;
3943         dpc->k_deferredctx = dpcctx;
3944         dpc->k_num = KDPC_CPU_DEFAULT;
3945         dpc->k_importance = KDPC_IMPORTANCE_MEDIUM;
3946         InitializeListHead((&dpc->k_dpclistentry));
3947 }
3948
3949 uint8_t
3950 KeInsertQueueDpc(dpc, sysarg1, sysarg2)
3951         kdpc                    *dpc;
3952         void                    *sysarg1;
3953         void                    *sysarg2;
3954 {
3955         kdpc_queue              *kq;
3956         uint8_t                 r;
3957         uint8_t                 irql;
3958
3959         if (dpc == NULL)
3960                 return (FALSE);
3961
3962         kq = kq_queues;
3963
3964 #ifdef NTOSKRNL_MULTIPLE_DPCS
3965         KeRaiseIrql(DISPATCH_LEVEL, &irql);
3966
3967         /*
3968          * By default, the DPC is queued to run on the same CPU
3969          * that scheduled it.
3970          */
3971
3972         if (dpc->k_num == KDPC_CPU_DEFAULT)
3973                 kq += curthread->td_oncpu;
3974         else
3975                 kq += dpc->k_num;
3976         KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3977 #else
3978         KeAcquireSpinLock(&kq->kq_lock, &irql);
3979 #endif
3980
3981         r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc);
3982         if (r == TRUE) {
3983                 dpc->k_sysarg1 = sysarg1;
3984                 dpc->k_sysarg2 = sysarg2;
3985         }
3986         KeReleaseSpinLock(&kq->kq_lock, irql);
3987
3988         if (r == FALSE)
3989                 return (r);
3990
3991         KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
3992
3993         return (r);
3994 }
3995
3996 uint8_t
3997 KeRemoveQueueDpc(dpc)
3998         kdpc                    *dpc;
3999 {
4000         kdpc_queue              *kq;
4001         uint8_t                 irql;
4002
4003         if (dpc == NULL)
4004                 return (FALSE);
4005
4006 #ifdef NTOSKRNL_MULTIPLE_DPCS
4007         KeRaiseIrql(DISPATCH_LEVEL, &irql);
4008
4009         kq = kq_queues + dpc->k_num;
4010
4011         KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
4012 #else
4013         kq = kq_queues;
4014         KeAcquireSpinLock(&kq->kq_lock, &irql);
4015 #endif
4016
4017         if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) {
4018                 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
4019                 KeLowerIrql(irql);
4020                 return (FALSE);
4021         }
4022
4023         RemoveEntryList((&dpc->k_dpclistentry));
4024         InitializeListHead((&dpc->k_dpclistentry));
4025
4026         KeReleaseSpinLock(&kq->kq_lock, irql);
4027
4028         return (TRUE);
4029 }
4030
4031 void
4032 KeSetImportanceDpc(dpc, imp)
4033         kdpc                    *dpc;
4034         uint32_t                imp;
4035 {
4036         if (imp != KDPC_IMPORTANCE_LOW &&
4037             imp != KDPC_IMPORTANCE_MEDIUM &&
4038             imp != KDPC_IMPORTANCE_HIGH)
4039                 return;
4040
4041         dpc->k_importance = (uint8_t)imp;
4042 }
4043
4044 void
4045 KeSetTargetProcessorDpc(kdpc *dpc, uint8_t cpu)
4046 {
4047         if (cpu > mp_ncpus)
4048                 return;
4049
4050         dpc->k_num = cpu;
4051 }
4052
4053 void
4054 KeFlushQueuedDpcs(void)
4055 {
4056         kdpc_queue              *kq;
4057         int                     i;
4058
4059         /*
4060          * Poke each DPC queue and wait
4061          * for them to drain.
4062          */
4063
4064 #ifdef NTOSKRNL_MULTIPLE_DPCS
4065         for (i = 0; i < mp_ncpus; i++) {
4066 #else
4067         for (i = 0; i < 1; i++) {
4068 #endif
4069                 kq = kq_queues + i;
4070                 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
4071                 KeWaitForSingleObject(&kq->kq_done, 0, 0, TRUE, NULL);
4072         }
4073 }
4074
4075 uint32_t
4076 KeGetCurrentProcessorNumber(void)
4077 {
4078         return ((uint32_t)curthread->td_oncpu);
4079 }
4080
4081 uint8_t
4082 KeSetTimerEx(timer, duetime, period, dpc)
4083         ktimer                  *timer;
4084         int64_t                 duetime;
4085         uint32_t                period;
4086         kdpc                    *dpc;
4087 {
4088         struct timeval          tv;
4089         uint64_t                curtime;
4090         uint8_t                 pending;
4091
4092         if (timer == NULL)
4093                 return (FALSE);
4094
4095         mtx_lock(&ntoskrnl_dispatchlock);
4096
4097         if (timer->k_header.dh_inserted == TRUE) {
4098                 ntoskrnl_remove_timer(timer);
4099 #ifdef NTOSKRNL_DEBUG_TIMERS
4100                 ntoskrnl_timer_cancels++;
4101 #endif
4102                 timer->k_header.dh_inserted = FALSE;
4103                 pending = TRUE;
4104         } else
4105                 pending = FALSE;
4106
4107         timer->k_duetime = duetime;
4108         timer->k_period = period;
4109         timer->k_header.dh_sigstate = FALSE;
4110         timer->k_dpc = dpc;
4111
4112         if (duetime < 0) {
4113                 tv.tv_sec = - (duetime) / 10000000;
4114                 tv.tv_usec = (- (duetime) / 10) -
4115                     (tv.tv_sec * 1000000);
4116         } else {
4117                 ntoskrnl_time(&curtime);
4118                 if (duetime < curtime)
4119                         tv.tv_sec = tv.tv_usec = 0;
4120                 else {
4121                         tv.tv_sec = ((duetime) - curtime) / 10000000;
4122                         tv.tv_usec = ((duetime) - curtime) / 10 -
4123                             (tv.tv_sec * 1000000);
4124                 }
4125         }
4126
4127         timer->k_header.dh_inserted = TRUE;
4128         ntoskrnl_insert_timer(timer, tvtohz(&tv));
4129 #ifdef NTOSKRNL_DEBUG_TIMERS
4130         ntoskrnl_timer_sets++;
4131 #endif
4132
4133         mtx_unlock(&ntoskrnl_dispatchlock);
4134
4135         return (pending);
4136 }
4137
4138 uint8_t
4139 KeSetTimer(timer, duetime, dpc)
4140         ktimer                  *timer;
4141         int64_t                 duetime;
4142         kdpc                    *dpc;
4143 {
4144         return (KeSetTimerEx(timer, duetime, 0, dpc));
4145 }
4146
4147 /*
4148  * The Windows DDK documentation seems to say that cancelling
4149  * a timer that has a DPC will result in the DPC also being
4150  * cancelled, but this isn't really the case.
4151  */
4152
4153 uint8_t
4154 KeCancelTimer(timer)
4155         ktimer                  *timer;
4156 {
4157         uint8_t                 pending;
4158
4159         if (timer == NULL)
4160                 return (FALSE);
4161
4162         mtx_lock(&ntoskrnl_dispatchlock);
4163
4164         pending = timer->k_header.dh_inserted;
4165
4166         if (timer->k_header.dh_inserted == TRUE) {
4167                 timer->k_header.dh_inserted = FALSE;
4168                 ntoskrnl_remove_timer(timer);
4169 #ifdef NTOSKRNL_DEBUG_TIMERS
4170                 ntoskrnl_timer_cancels++;
4171 #endif
4172         }
4173
4174         mtx_unlock(&ntoskrnl_dispatchlock);
4175
4176         return (pending);
4177 }
4178
4179 uint8_t
4180 KeReadStateTimer(timer)
4181         ktimer                  *timer;
4182 {
4183         return (timer->k_header.dh_sigstate);
4184 }
4185
4186 static int32_t
4187 KeDelayExecutionThread(uint8_t wait_mode, uint8_t alertable, int64_t *interval)
4188 {
4189         ktimer                  timer;
4190
4191         if (wait_mode != 0)
4192                 panic("invalid wait_mode %d", wait_mode);
4193
4194         KeInitializeTimer(&timer);
4195         KeSetTimer(&timer, *interval, NULL);
4196         KeWaitForSingleObject(&timer, 0, 0, alertable, NULL);
4197
4198         return STATUS_SUCCESS;
4199 }
4200
4201 static uint64_t
4202 KeQueryInterruptTime(void)
4203 {
4204         int ticks;
4205         struct timeval tv;
4206
4207         getmicrouptime(&tv);
4208
4209         ticks = tvtohz(&tv);
4210
4211         return ticks * ((10000000 + hz - 1) / hz);
4212 }
4213
4214 static struct thread *
4215 KeGetCurrentThread(void)
4216 {
4217
4218         return curthread;
4219 }
4220
4221 static int32_t
4222 KeSetPriorityThread(td, pri)
4223         struct thread   *td;
4224         int32_t         pri;
4225 {
4226         int32_t old;
4227
4228         if (td == NULL)
4229                 return LOW_REALTIME_PRIORITY;
4230
4231         if (td->td_priority <= PRI_MIN_KERN)
4232                 old = HIGH_PRIORITY;
4233         else if (td->td_priority >= PRI_MAX_KERN)
4234                 old = LOW_PRIORITY;
4235         else
4236                 old = LOW_REALTIME_PRIORITY;
4237
4238         thread_lock(td);
4239         if (pri == HIGH_PRIORITY)
4240                 sched_prio(td, PRI_MIN_KERN);
4241         if (pri == LOW_REALTIME_PRIORITY)
4242                 sched_prio(td, PRI_MIN_KERN + (PRI_MAX_KERN - PRI_MIN_KERN) / 2);
4243         if (pri == LOW_PRIORITY)
4244                 sched_prio(td, PRI_MAX_KERN);
4245         thread_unlock(td);
4246
4247         return old;
4248 }
4249
4250 static void
4251 dummy()
4252 {
4253         printf("ntoskrnl dummy called...\n");
4254 }
4255
4256
4257 image_patch_table ntoskrnl_functbl[] = {
4258         IMPORT_SFUNC(RtlZeroMemory, 2),
4259         IMPORT_SFUNC(RtlSecureZeroMemory, 2),
4260         IMPORT_SFUNC(RtlFillMemory, 3),
4261         IMPORT_SFUNC(RtlMoveMemory, 3),
4262         IMPORT_SFUNC(RtlCharToInteger, 3),
4263         IMPORT_SFUNC(RtlCopyMemory, 3),
4264         IMPORT_SFUNC(RtlCopyString, 2),
4265         IMPORT_SFUNC(RtlCompareMemory, 3),
4266         IMPORT_SFUNC(RtlEqualUnicodeString, 3),
4267         IMPORT_SFUNC(RtlCopyUnicodeString, 2),
4268         IMPORT_SFUNC(RtlUnicodeStringToAnsiString, 3),
4269         IMPORT_SFUNC(RtlAnsiStringToUnicodeString, 3),
4270         IMPORT_SFUNC(RtlInitAnsiString, 2),
4271         IMPORT_SFUNC_MAP(RtlInitString, RtlInitAnsiString, 2),
4272         IMPORT_SFUNC(RtlInitUnicodeString, 2),
4273         IMPORT_SFUNC(RtlFreeAnsiString, 1),
4274         IMPORT_SFUNC(RtlFreeUnicodeString, 1),
4275         IMPORT_SFUNC(RtlUnicodeStringToInteger, 3),
4276         IMPORT_CFUNC(sprintf, 0),
4277         IMPORT_CFUNC(vsprintf, 0),
4278         IMPORT_CFUNC_MAP(_snprintf, snprintf, 0),
4279         IMPORT_CFUNC_MAP(_vsnprintf, vsnprintf, 0),
4280         IMPORT_CFUNC(DbgPrint, 0),
4281         IMPORT_SFUNC(DbgBreakPoint, 0),
4282         IMPORT_SFUNC(KeBugCheckEx, 5),
4283         IMPORT_CFUNC(strncmp, 0),
4284         IMPORT_CFUNC(strcmp, 0),
4285         IMPORT_CFUNC_MAP(stricmp, strcasecmp, 0),
4286         IMPORT_CFUNC(strncpy, 0),
4287         IMPORT_CFUNC(strcpy, 0),
4288         IMPORT_CFUNC(strlen, 0),
4289         IMPORT_CFUNC_MAP(toupper, ntoskrnl_toupper, 0),
4290         IMPORT_CFUNC_MAP(tolower, ntoskrnl_tolower, 0),
4291         IMPORT_CFUNC_MAP(strstr, ntoskrnl_strstr, 0),
4292         IMPORT_CFUNC_MAP(strncat, ntoskrnl_strncat, 0),
4293         IMPORT_CFUNC_MAP(strchr, index, 0),
4294         IMPORT_CFUNC_MAP(strrchr, rindex, 0),
4295         IMPORT_CFUNC(memcpy, 0),
4296         IMPORT_CFUNC_MAP(memmove, ntoskrnl_memmove, 0),
4297         IMPORT_CFUNC_MAP(memset, ntoskrnl_memset, 0),
4298         IMPORT_CFUNC_MAP(memchr, ntoskrnl_memchr, 0),
4299         IMPORT_SFUNC(IoAllocateDriverObjectExtension, 4),
4300         IMPORT_SFUNC(IoGetDriverObjectExtension, 2),
4301         IMPORT_FFUNC(IofCallDriver, 2),
4302         IMPORT_FFUNC(IofCompleteRequest, 2),
4303         IMPORT_SFUNC(IoAcquireCancelSpinLock, 1),
4304         IMPORT_SFUNC(IoReleaseCancelSpinLock, 1),
4305         IMPORT_SFUNC(IoCancelIrp, 1),
4306         IMPORT_SFUNC(IoConnectInterrupt, 11),
4307         IMPORT_SFUNC(IoDisconnectInterrupt, 1),
4308         IMPORT_SFUNC(IoCreateDevice, 7),
4309         IMPORT_SFUNC(IoDeleteDevice, 1),
4310         IMPORT_SFUNC(IoGetAttachedDevice, 1),
4311         IMPORT_SFUNC(IoAttachDeviceToDeviceStack, 2),
4312         IMPORT_SFUNC(IoDetachDevice, 1),
4313         IMPORT_SFUNC(IoBuildSynchronousFsdRequest, 7),
4314         IMPORT_SFUNC(IoBuildAsynchronousFsdRequest, 6),
4315         IMPORT_SFUNC(IoBuildDeviceIoControlRequest, 9),
4316         IMPORT_SFUNC(IoAllocateIrp, 2),
4317         IMPORT_SFUNC(IoReuseIrp, 2),
4318         IMPORT_SFUNC(IoMakeAssociatedIrp, 2),
4319         IMPORT_SFUNC(IoFreeIrp, 1),
4320         IMPORT_SFUNC(IoInitializeIrp, 3),
4321         IMPORT_SFUNC(KeAcquireInterruptSpinLock, 1),
4322         IMPORT_SFUNC(KeReleaseInterruptSpinLock, 2),
4323         IMPORT_SFUNC(KeSynchronizeExecution, 3),
4324         IMPORT_SFUNC(KeWaitForSingleObject, 5),
4325         IMPORT_SFUNC(KeWaitForMultipleObjects, 8),
4326         IMPORT_SFUNC(_allmul, 4),
4327         IMPORT_SFUNC(_alldiv, 4),
4328         IMPORT_SFUNC(_allrem, 4),
4329         IMPORT_RFUNC(_allshr, 0),
4330         IMPORT_RFUNC(_allshl, 0),
4331         IMPORT_SFUNC(_aullmul, 4),
4332         IMPORT_SFUNC(_aulldiv, 4),
4333         IMPORT_SFUNC(_aullrem, 4),
4334         IMPORT_RFUNC(_aullshr, 0),
4335         IMPORT_RFUNC(_aullshl, 0),
4336         IMPORT_CFUNC(atoi, 0),
4337         IMPORT_CFUNC(atol, 0),
4338         IMPORT_CFUNC(rand, 0),
4339         IMPORT_CFUNC(srand, 0),
4340         IMPORT_SFUNC(WRITE_REGISTER_USHORT, 2),
4341         IMPORT_SFUNC(READ_REGISTER_USHORT, 1),
4342         IMPORT_SFUNC(WRITE_REGISTER_ULONG, 2),
4343         IMPORT_SFUNC(READ_REGISTER_ULONG, 1),
4344         IMPORT_SFUNC(READ_REGISTER_UCHAR, 1),
4345         IMPORT_SFUNC(WRITE_REGISTER_UCHAR, 2),
4346         IMPORT_SFUNC(ExInitializePagedLookasideList, 7),
4347         IMPORT_SFUNC(ExDeletePagedLookasideList, 1),
4348         IMPORT_SFUNC(ExInitializeNPagedLookasideList, 7),
4349         IMPORT_SFUNC(ExDeleteNPagedLookasideList, 1),
4350         IMPORT_FFUNC(InterlockedPopEntrySList, 1),
4351         IMPORT_FFUNC(InitializeSListHead, 1),
4352         IMPORT_FFUNC(InterlockedPushEntrySList, 2),
4353         IMPORT_SFUNC(ExQueryDepthSList, 1),
4354         IMPORT_FFUNC_MAP(ExpInterlockedPopEntrySList,
4355                 InterlockedPopEntrySList, 1),
4356         IMPORT_FFUNC_MAP(ExpInterlockedPushEntrySList,
4357                 InterlockedPushEntrySList, 2),
4358         IMPORT_FFUNC(ExInterlockedPopEntrySList, 2),
4359         IMPORT_FFUNC(ExInterlockedPushEntrySList, 3),
4360         IMPORT_SFUNC(ExAllocatePoolWithTag, 3),
4361         IMPORT_SFUNC(ExFreePoolWithTag, 2),
4362         IMPORT_SFUNC(ExFreePool, 1),
4363 #ifdef __i386__
4364         IMPORT_FFUNC(KefAcquireSpinLockAtDpcLevel, 1),
4365         IMPORT_FFUNC(KefReleaseSpinLockFromDpcLevel,1),
4366         IMPORT_FFUNC(KeAcquireSpinLockRaiseToDpc, 1),
4367 #else
4368         /*
4369          * For AMD64, we can get away with just mapping
4370          * KeAcquireSpinLockRaiseToDpc() directly to KfAcquireSpinLock()
4371          * because the calling conventions end up being the same.
4372          * On i386, we have to be careful because KfAcquireSpinLock()
4373          * is _fastcall but KeAcquireSpinLockRaiseToDpc() isn't.
4374          */
4375         IMPORT_SFUNC(KeAcquireSpinLockAtDpcLevel, 1),
4376         IMPORT_SFUNC(KeReleaseSpinLockFromDpcLevel, 1),
4377         IMPORT_SFUNC_MAP(KeAcquireSpinLockRaiseToDpc, KfAcquireSpinLock, 1),
4378 #endif
4379         IMPORT_SFUNC_MAP(KeReleaseSpinLock, KfReleaseSpinLock, 1),
4380         IMPORT_FFUNC(InterlockedIncrement, 1),
4381         IMPORT_FFUNC(InterlockedDecrement, 1),
4382         IMPORT_FFUNC(InterlockedExchange, 2),
4383         IMPORT_FFUNC(ExInterlockedAddLargeStatistic, 2),
4384         IMPORT_SFUNC(IoAllocateMdl, 5),
4385         IMPORT_SFUNC(IoFreeMdl, 1),
4386         IMPORT_SFUNC(MmAllocateContiguousMemory, 2 + 1),
4387         IMPORT_SFUNC(MmAllocateContiguousMemorySpecifyCache, 5 + 3),
4388         IMPORT_SFUNC(MmFreeContiguousMemory, 1),
4389         IMPORT_SFUNC(MmFreeContiguousMemorySpecifyCache, 3),
4390         IMPORT_SFUNC(MmSizeOfMdl, 1),
4391         IMPORT_SFUNC(MmMapLockedPages, 2),
4392         IMPORT_SFUNC(MmMapLockedPagesSpecifyCache, 6),
4393         IMPORT_SFUNC(MmUnmapLockedPages, 2),
4394         IMPORT_SFUNC(MmBuildMdlForNonPagedPool, 1),
4395         IMPORT_SFUNC(MmGetPhysicalAddress, 1),
4396         IMPORT_SFUNC(MmGetSystemRoutineAddress, 1),
4397         IMPORT_SFUNC(MmIsAddressValid, 1),
4398         IMPORT_SFUNC(MmMapIoSpace, 3 + 1),
4399         IMPORT_SFUNC(MmUnmapIoSpace, 2),
4400         IMPORT_SFUNC(KeInitializeSpinLock, 1),
4401         IMPORT_SFUNC(IoIsWdmVersionAvailable, 2),
4402         IMPORT_SFUNC(IoOpenDeviceRegistryKey, 4),
4403         IMPORT_SFUNC(IoGetDeviceObjectPointer, 4),
4404         IMPORT_SFUNC(IoGetDeviceProperty, 5),
4405         IMPORT_SFUNC(IoAllocateWorkItem, 1),
4406         IMPORT_SFUNC(IoFreeWorkItem, 1),
4407         IMPORT_SFUNC(IoQueueWorkItem, 4),
4408         IMPORT_SFUNC(ExQueueWorkItem, 2),
4409         IMPORT_SFUNC(ntoskrnl_workitem, 2),
4410         IMPORT_SFUNC(KeInitializeMutex, 2),
4411         IMPORT_SFUNC(KeReleaseMutex, 2),
4412         IMPORT_SFUNC(KeReadStateMutex, 1),
4413         IMPORT_SFUNC(KeInitializeEvent, 3),
4414         IMPORT_SFUNC(KeSetEvent, 3),
4415         IMPORT_SFUNC(KeResetEvent, 1),
4416         IMPORT_SFUNC(KeClearEvent, 1),
4417         IMPORT_SFUNC(KeReadStateEvent, 1),
4418         IMPORT_SFUNC(KeInitializeTimer, 1),
4419         IMPORT_SFUNC(KeInitializeTimerEx, 2),
4420         IMPORT_SFUNC(KeSetTimer, 3),
4421         IMPORT_SFUNC(KeSetTimerEx, 4),
4422         IMPORT_SFUNC(KeCancelTimer, 1),
4423         IMPORT_SFUNC(KeReadStateTimer, 1),
4424         IMPORT_SFUNC(KeInitializeDpc, 3),
4425         IMPORT_SFUNC(KeInsertQueueDpc, 3),
4426         IMPORT_SFUNC(KeRemoveQueueDpc, 1),
4427         IMPORT_SFUNC(KeSetImportanceDpc, 2),
4428         IMPORT_SFUNC(KeSetTargetProcessorDpc, 2),
4429         IMPORT_SFUNC(KeFlushQueuedDpcs, 0),
4430         IMPORT_SFUNC(KeGetCurrentProcessorNumber, 1),
4431         IMPORT_SFUNC(ObReferenceObjectByHandle, 6),
4432         IMPORT_FFUNC(ObfDereferenceObject, 1),
4433         IMPORT_SFUNC(ZwClose, 1),
4434         IMPORT_SFUNC(PsCreateSystemThread, 7),
4435         IMPORT_SFUNC(PsTerminateSystemThread, 1),
4436         IMPORT_SFUNC(IoWMIRegistrationControl, 2),
4437         IMPORT_SFUNC(WmiQueryTraceInformation, 5),
4438         IMPORT_CFUNC(WmiTraceMessage, 0),
4439         IMPORT_SFUNC(KeQuerySystemTime, 1),
4440         IMPORT_CFUNC(KeTickCount, 0),
4441         IMPORT_SFUNC(KeDelayExecutionThread, 3),
4442         IMPORT_SFUNC(KeQueryInterruptTime, 0),
4443         IMPORT_SFUNC(KeGetCurrentThread, 0),
4444         IMPORT_SFUNC(KeSetPriorityThread, 2),
4445
4446         /*
4447          * This last entry is a catch-all for any function we haven't
4448          * implemented yet. The PE import list patching routine will
4449          * use it for any function that doesn't have an explicit match
4450          * in this table.
4451          */
4452
4453         { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL },
4454
4455         /* End of list. */
4456
4457         { NULL, NULL, NULL }
4458 };