sys/compat/ndis/subr_ntoskrnl.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-4-Clause
   3  *
   4  * Copyright (c) 2003
   5  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. All advertising materials mentioning features or use of this software
  16  *    must display the following acknowledgement:
  17  *      This product includes software developed by Bill Paul.
  18  * 4. Neither the name of the author nor the names of any co-contributors
  19  *    may be used to endorse or promote products derived from this software
  20  *    without specific prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  26  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  32  * THE POSSIBILITY OF SUCH DAMAGE.
  33  */
  34
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include <sys/ctype.h>
  39 #include <sys/unistd.h>
  40 #include <sys/param.h>
  41 #include <sys/types.h>
  42 #include <sys/errno.h>
  43 #include <sys/systm.h>
  44 #include <sys/malloc.h>
  45 #include <sys/lock.h>
  46 #include <sys/mutex.h>
  47
  48 #include <sys/callout.h>
  49 #include <sys/kdb.h>
  50 #include <sys/kernel.h>
  51 #include <sys/proc.h>
  52 #include <sys/condvar.h>
  53 #include <sys/kthread.h>
  54 #include <sys/module.h>
  55 #include <sys/smp.h>
  56 #include <sys/sched.h>
  57 #include <sys/sysctl.h>
  58
  59 #include <machine/atomic.h>
  60 #include <machine/bus.h>
  61 #include <machine/stdarg.h>
  62 #include <machine/resource.h>
  63
  64 #include <sys/bus.h>
  65 #include <sys/rman.h>
  66
  67 #include <vm/vm.h>
  68 #include <vm/vm_param.h>
  69 #include <vm/pmap.h>
  70 #include <vm/uma.h>
  71 #include <vm/vm_kern.h>
  72 #include <vm/vm_map.h>
  73 #include <vm/vm_extern.h>
  74
  75 #include <compat/ndis/pe_var.h>
  76 #include <compat/ndis/cfg_var.h>
  77 #include <compat/ndis/resource_var.h>
  78 #include <compat/ndis/ntoskrnl_var.h>
  79 #include <compat/ndis/hal_var.h>
  80 #include <compat/ndis/ndis_var.h>
  81
  82 #ifdef NTOSKRNL_DEBUG_TIMERS
  83 static int sysctl_show_timers(SYSCTL_HANDLER_ARGS);
  84
  85 SYSCTL_PROC(_debug, OID_AUTO, ntoskrnl_timers, CTLTYPE_INT | CTLFLAG_RW,
  86     NULL, 0, sysctl_show_timers, "I",
  87     "Show ntoskrnl timer stats");
  88 #endif
  89
  90 struct kdpc_queue {
  91         list_entry              kq_disp;
  92         struct thread           *kq_td;
  93         int                     kq_cpu;
  94         int                     kq_exit;
  95         int                     kq_running;
  96         kspin_lock              kq_lock;
  97         nt_kevent               kq_proc;
  98         nt_kevent               kq_done;
  99 };
 100
 101 typedef struct kdpc_queue kdpc_queue;
 102
 103 struct wb_ext {
 104         struct cv               we_cv;
 105         struct thread           *we_td;
 106 };
 107
 108 typedef struct wb_ext wb_ext;
 109
 110 #define NTOSKRNL_TIMEOUTS       256
 111 #ifdef NTOSKRNL_DEBUG_TIMERS
 112 static uint64_t ntoskrnl_timer_fires;
 113 static uint64_t ntoskrnl_timer_sets;
 114 static uint64_t ntoskrnl_timer_reloads;
 115 static uint64_t ntoskrnl_timer_cancels;
 116 #endif
 117
 118 struct callout_entry {
 119         struct callout          ce_callout;
 120         list_entry              ce_list;
 121 };
 122
 123 typedef struct callout_entry callout_entry;
 124
 125 static struct list_entry ntoskrnl_calllist;
 126 static struct mtx ntoskrnl_calllock;
 127 struct kuser_shared_data kuser_shared_data;
 128
 129 static struct list_entry ntoskrnl_intlist;
 130 static kspin_lock ntoskrnl_intlock;
 131
 132 static uint8_t RtlEqualUnicodeString(unicode_string *,
 133         unicode_string *, uint8_t);
 134 static void RtlCopyString(ansi_string *, const ansi_string *);
 135 static void RtlCopyUnicodeString(unicode_string *,
 136         unicode_string *);
 137 static irp *IoBuildSynchronousFsdRequest(uint32_t, device_object *,
 138          void *, uint32_t, uint64_t *, nt_kevent *, io_status_block *);
 139 static irp *IoBuildAsynchronousFsdRequest(uint32_t,
 140         device_object *, void *, uint32_t, uint64_t *, io_status_block *);
 141 static irp *IoBuildDeviceIoControlRequest(uint32_t,
 142         device_object *, void *, uint32_t, void *, uint32_t,
 143         uint8_t, nt_kevent *, io_status_block *);
 144 static irp *IoAllocateIrp(uint8_t, uint8_t);
 145 static void IoReuseIrp(irp *, uint32_t);
 146 static void IoFreeIrp(irp *);
 147 static void IoInitializeIrp(irp *, uint16_t, uint8_t);
 148 static irp *IoMakeAssociatedIrp(irp *, uint8_t);
 149 static uint32_t KeWaitForMultipleObjects(uint32_t,
 150         nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t,
 151         int64_t *, wait_block *);
 152 static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t);
 153 static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *);
 154 static void ntoskrnl_satisfy_multiple_waits(wait_block *);
 155 static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *);
 156 static void ntoskrnl_insert_timer(ktimer *, int);
 157 static void ntoskrnl_remove_timer(ktimer *);
 158 #ifdef NTOSKRNL_DEBUG_TIMERS
 159 static void ntoskrnl_show_timers(void);
 160 #endif
 161 static void ntoskrnl_timercall(void *);
 162 static void ntoskrnl_dpc_thread(void *);
 163 static void ntoskrnl_destroy_dpc_threads(void);
 164 static void ntoskrnl_destroy_workitem_threads(void);
 165 static void ntoskrnl_workitem_thread(void *);
 166 static void ntoskrnl_workitem(device_object *, void *);
 167 static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int);
 168 static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int);
 169 static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *);
 170 static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t);
 171 static uint16_t READ_REGISTER_USHORT(uint16_t *);
 172 static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t);
 173 static uint32_t READ_REGISTER_ULONG(uint32_t *);
 174 static void WRITE_REGISTER_UCHAR(uint8_t *, uint8_t);
 175 static uint8_t READ_REGISTER_UCHAR(uint8_t *);
 176 static int64_t _allmul(int64_t, int64_t);
 177 static int64_t _alldiv(int64_t, int64_t);
 178 static int64_t _allrem(int64_t, int64_t);
 179 static int64_t _allshr(int64_t, uint8_t);
 180 static int64_t _allshl(int64_t, uint8_t);
 181 static uint64_t _aullmul(uint64_t, uint64_t);
 182 static uint64_t _aulldiv(uint64_t, uint64_t);
 183 static uint64_t _aullrem(uint64_t, uint64_t);
 184 static uint64_t _aullshr(uint64_t, uint8_t);
 185 static uint64_t _aullshl(uint64_t, uint8_t);
 186 static slist_entry *ntoskrnl_pushsl(slist_header *, slist_entry *);
 187 static void InitializeSListHead(slist_header *);
 188 static slist_entry *ntoskrnl_popsl(slist_header *);
 189 static void ExFreePoolWithTag(void *, uint32_t);
 190 static void ExInitializePagedLookasideList(paged_lookaside_list *,
 191         lookaside_alloc_func *, lookaside_free_func *,
 192         uint32_t, size_t, uint32_t, uint16_t);
 193 static void ExDeletePagedLookasideList(paged_lookaside_list *);
 194 static void ExInitializeNPagedLookasideList(npaged_lookaside_list *,
 195         lookaside_alloc_func *, lookaside_free_func *,
 196         uint32_t, size_t, uint32_t, uint16_t);
 197 static void ExDeleteNPagedLookasideList(npaged_lookaside_list *);
 198 static slist_entry
 199         *ExInterlockedPushEntrySList(slist_header *,
 200         slist_entry *, kspin_lock *);
 201 static slist_entry
 202         *ExInterlockedPopEntrySList(slist_header *, kspin_lock *);
 203 static uint32_t InterlockedIncrement(volatile uint32_t *);
 204 static uint32_t InterlockedDecrement(volatile uint32_t *);
 205 static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t);
 206 static void *MmAllocateContiguousMemory(uint32_t, uint64_t);
 207 static void *MmAllocateContiguousMemorySpecifyCache(uint32_t,
 208         uint64_t, uint64_t, uint64_t, enum nt_caching_type);
 209 static void MmFreeContiguousMemory(void *);
 210 static void MmFreeContiguousMemorySpecifyCache(void *, uint32_t,
 211         enum nt_caching_type);
 212 static uint32_t MmSizeOfMdl(void *, size_t);
 213 static void *MmMapLockedPages(mdl *, uint8_t);
 214 static void *MmMapLockedPagesSpecifyCache(mdl *,
 215         uint8_t, uint32_t, void *, uint32_t, uint32_t);
 216 static void MmUnmapLockedPages(void *, mdl *);
 217 static device_t ntoskrnl_finddev(device_t, uint64_t, struct resource **);
 218 static void RtlZeroMemory(void *, size_t);
 219 static void RtlSecureZeroMemory(void *, size_t);
 220 static void RtlFillMemory(void *, size_t, uint8_t);
 221 static void RtlMoveMemory(void *, const void *, size_t);
 222 static ndis_status RtlCharToInteger(const char *, uint32_t, uint32_t *);
 223 static void RtlCopyMemory(void *, const void *, size_t);
 224 static size_t RtlCompareMemory(const void *, const void *, size_t);
 225 static ndis_status RtlUnicodeStringToInteger(unicode_string *,
 226         uint32_t, uint32_t *);
 227 static int atoi (const char *);
 228 static long atol (const char *);
 229 static int rand(void);
 230 static void srand(unsigned int);
 231 static void KeQuerySystemTime(uint64_t *);
 232 static uint32_t KeTickCount(void);
 233 static uint8_t IoIsWdmVersionAvailable(uint8_t, uint8_t);
 234 static int32_t IoOpenDeviceRegistryKey(struct device_object *, uint32_t,
 235     uint32_t, void **);
 236 static void ntoskrnl_thrfunc(void *);
 237 static ndis_status PsCreateSystemThread(ndis_handle *,
 238         uint32_t, void *, ndis_handle, void *, void *, void *);
 239 static ndis_status PsTerminateSystemThread(ndis_status);
 240 static ndis_status IoGetDeviceObjectPointer(unicode_string *,
 241         uint32_t, void *, device_object *);
 242 static ndis_status IoGetDeviceProperty(device_object *, uint32_t,
 243         uint32_t, void *, uint32_t *);
 244 static void KeInitializeMutex(kmutant *, uint32_t);
 245 static uint32_t KeReleaseMutex(kmutant *, uint8_t);
 246 static uint32_t KeReadStateMutex(kmutant *);
 247 static ndis_status ObReferenceObjectByHandle(ndis_handle,
 248         uint32_t, void *, uint8_t, void **, void **);
 249 static void ObfDereferenceObject(void *);
 250 static uint32_t ZwClose(ndis_handle);
 251 static uint32_t WmiQueryTraceInformation(uint32_t, void *, uint32_t,
 252         uint32_t, void *);
 253 static uint32_t WmiTraceMessage(uint64_t, uint32_t, void *, uint16_t, ...);
 254 static uint32_t IoWMIRegistrationControl(device_object *, uint32_t);
 255 static void *ntoskrnl_memset(void *, int, size_t);
 256 static void *ntoskrnl_memmove(void *, void *, size_t);
 257 static void *ntoskrnl_memchr(void *, unsigned char, size_t);
 258 static char *ntoskrnl_strstr(char *, char *);
 259 static char *ntoskrnl_strncat(char *, char *, size_t);
 260 static int ntoskrnl_toupper(int);
 261 static int ntoskrnl_tolower(int);
 262 static funcptr ntoskrnl_findwrap(funcptr);
 263 static uint32_t DbgPrint(char *, ...);
 264 static void DbgBreakPoint(void);
 265 static void KeBugCheckEx(uint32_t, u_long, u_long, u_long, u_long);
 266 static int32_t KeDelayExecutionThread(uint8_t, uint8_t, int64_t *);
 267 static int32_t KeSetPriorityThread(struct thread *, int32_t);
 268 static void dummy(void);
 269
 270 static struct mtx ntoskrnl_dispatchlock;
 271 static struct mtx ntoskrnl_interlock;
 272 static kspin_lock ntoskrnl_cancellock;
 273 static int ntoskrnl_kth = 0;
 274 static struct nt_objref_head ntoskrnl_reflist;
 275 static uma_zone_t mdl_zone;
 276 static uma_zone_t iw_zone;
 277 static struct kdpc_queue *kq_queues;
 278 static struct kdpc_queue *wq_queues;
 279 static int wq_idx = 0;
 280
 281 int
 282 ntoskrnl_libinit()
 283 {
 284         image_patch_table       *patch;
 285         int                     error;
 286         struct proc             *p;
 287         kdpc_queue              *kq;
 288         callout_entry           *e;
 289         int                     i;
 290
 291         mtx_init(&ntoskrnl_dispatchlock,
 292             "ntoskrnl dispatch lock", MTX_NDIS_LOCK, MTX_DEF|MTX_RECURSE);
 293         mtx_init(&ntoskrnl_interlock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
 294         KeInitializeSpinLock(&ntoskrnl_cancellock);
 295         KeInitializeSpinLock(&ntoskrnl_intlock);
 296         TAILQ_INIT(&ntoskrnl_reflist);
 297
 298         InitializeListHead(&ntoskrnl_calllist);
 299         InitializeListHead(&ntoskrnl_intlist);
 300         mtx_init(&ntoskrnl_calllock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
 301
 302         kq_queues = ExAllocatePoolWithTag(NonPagedPool,
 303 #ifdef NTOSKRNL_MULTIPLE_DPCS
 304             sizeof(kdpc_queue) * mp_ncpus, 0);
 305 #else
 306             sizeof(kdpc_queue), 0);
 307 #endif
 308
 309         if (kq_queues == NULL)
 310                 return (ENOMEM);
 311
 312         wq_queues = ExAllocatePoolWithTag(NonPagedPool,
 313             sizeof(kdpc_queue) * WORKITEM_THREADS, 0);
 314
 315         if (wq_queues == NULL)
 316                 return (ENOMEM);
 317
 318 #ifdef NTOSKRNL_MULTIPLE_DPCS
 319         bzero((char *)kq_queues, sizeof(kdpc_queue) * mp_ncpus);
 320 #else
 321         bzero((char *)kq_queues, sizeof(kdpc_queue));
 322 #endif
 323         bzero((char *)wq_queues, sizeof(kdpc_queue) * WORKITEM_THREADS);
 324
 325         /*
 326          * Launch the DPC threads.
 327          */
 328
 329 #ifdef NTOSKRNL_MULTIPLE_DPCS
 330         for (i = 0; i < mp_ncpus; i++) {
 331 #else
 332         for (i = 0; i < 1; i++) {
 333 #endif
 334                 kq = kq_queues + i;
 335                 kq->kq_cpu = i;
 336                 error = kproc_create(ntoskrnl_dpc_thread, kq, &p,
 337                     RFHIGHPID, NDIS_KSTACK_PAGES, "Windows DPC %d", i);
 338                 if (error)
 339                         panic("failed to launch DPC thread");
 340         }
 341
 342         /*
 343          * Launch the workitem threads.
 344          */
 345
 346         for (i = 0; i < WORKITEM_THREADS; i++) {
 347                 kq = wq_queues + i;
 348                 error = kproc_create(ntoskrnl_workitem_thread, kq, &p,
 349                     RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Workitem %d", i);
 350                 if (error)
 351                         panic("failed to launch workitem thread");
 352         }
 353
 354         patch = ntoskrnl_functbl;
 355         while (patch->ipt_func != NULL) {
 356                 windrv_wrap((funcptr)patch->ipt_func,
 357                     (funcptr *)&patch->ipt_wrap,
 358                     patch->ipt_argcnt, patch->ipt_ftype);
 359                 patch++;
 360         }
 361
 362         for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) {
 363                 e = ExAllocatePoolWithTag(NonPagedPool,
 364                     sizeof(callout_entry), 0);
 365                 if (e == NULL)
 366                         panic("failed to allocate timeouts");
 367                 mtx_lock_spin(&ntoskrnl_calllock);
 368                 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
 369                 mtx_unlock_spin(&ntoskrnl_calllock);
 370         }
 371
 372         /*
 373          * MDLs are supposed to be variable size (they describe
 374          * buffers containing some number of pages, but we don't
 375          * know ahead of time how many pages that will be). But
 376          * always allocating them off the heap is very slow. As
 377          * a compromise, we create an MDL UMA zone big enough to
 378          * handle any buffer requiring up to 16 pages, and we
 379          * use those for any MDLs for buffers of 16 pages or less
 380          * in size. For buffers larger than that (which we assume
 381          * will be few and far between, we allocate the MDLs off
 382          * the heap.
 383          */
 384
 385         mdl_zone = uma_zcreate("Windows MDL", MDL_ZONE_SIZE,
 386             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 387
 388         iw_zone = uma_zcreate("Windows WorkItem", sizeof(io_workitem),
 389             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 390
 391         return (0);
 392 }
 393
 394 int
 395 ntoskrnl_libfini()
 396 {
 397         image_patch_table       *patch;
 398         callout_entry           *e;
 399         list_entry              *l;
 400
 401         patch = ntoskrnl_functbl;
 402         while (patch->ipt_func != NULL) {
 403                 windrv_unwrap(patch->ipt_wrap);
 404                 patch++;
 405         }
 406
 407         /* Stop the workitem queues. */
 408         ntoskrnl_destroy_workitem_threads();
 409         /* Stop the DPC queues. */
 410         ntoskrnl_destroy_dpc_threads();
 411
 412         ExFreePool(kq_queues);
 413         ExFreePool(wq_queues);
 414
 415         uma_zdestroy(mdl_zone);
 416         uma_zdestroy(iw_zone);
 417
 418         mtx_lock_spin(&ntoskrnl_calllock);
 419         while(!IsListEmpty(&ntoskrnl_calllist)) {
 420                 l = RemoveHeadList(&ntoskrnl_calllist);
 421                 e = CONTAINING_RECORD(l, callout_entry, ce_list);
 422                 mtx_unlock_spin(&ntoskrnl_calllock);
 423                 ExFreePool(e);
 424                 mtx_lock_spin(&ntoskrnl_calllock);
 425         }
 426         mtx_unlock_spin(&ntoskrnl_calllock);
 427
 428         mtx_destroy(&ntoskrnl_dispatchlock);
 429         mtx_destroy(&ntoskrnl_interlock);
 430         mtx_destroy(&ntoskrnl_calllock);
 431
 432         return (0);
 433 }
 434
 435 /*
 436  * We need to be able to reference this externally from the wrapper;
 437  * GCC only generates a local implementation of memset.
 438  */
 439 static void *
 440 ntoskrnl_memset(buf, ch, size)
 441         void                    *buf;
 442         int                     ch;
 443         size_t                  size;
 444 {
 445         return (memset(buf, ch, size));
 446 }
 447
 448 static void *
 449 ntoskrnl_memmove(dst, src, size)
 450         void                    *src;
 451         void                    *dst;
 452         size_t                  size;
 453 {
 454         bcopy(src, dst, size);
 455         return (dst);
 456 }
 457
 458 static void *
 459 ntoskrnl_memchr(void *buf, unsigned char ch, size_t len)
 460 {
 461         if (len != 0) {
 462                 unsigned char *p = buf;
 463
 464                 do {
 465                         if (*p++ == ch)
 466                                 return (p - 1);
 467                 } while (--len != 0);
 468         }
 469         return (NULL);
 470 }
 471
 472 static char *
 473 ntoskrnl_strstr(s, find)
 474         char *s, *find;
 475 {
 476         char c, sc;
 477         size_t len;
 478
 479         if ((c = *find++) != 0) {
 480                 len = strlen(find);
 481                 do {
 482                         do {
 483                                 if ((sc = *s++) == 0)
 484                                         return (NULL);
 485                         } while (sc != c);
 486                 } while (strncmp(s, find, len) != 0);
 487                 s--;
 488         }
 489         return ((char *)s);
 490 }
 491
 492 /* Taken from libc */
 493 static char *
 494 ntoskrnl_strncat(dst, src, n)
 495         char            *dst;
 496         char            *src;
 497         size_t          n;
 498 {
 499         if (n != 0) {
 500                 char *d = dst;
 501                 const char *s = src;
 502
 503                 while (*d != 0)
 504                         d++;
 505                 do {
 506                         if ((*d = *s++) == 0)
 507                                 break;
 508                         d++;
 509                 } while (--n != 0);
 510                 *d = 0;
 511         }
 512         return (dst);
 513 }
 514
 515 static int
 516 ntoskrnl_toupper(c)
 517         int                     c;
 518 {
 519         return (toupper(c));
 520 }
 521
 522 static int
 523 ntoskrnl_tolower(c)
 524         int                     c;
 525 {
 526         return (tolower(c));
 527 }
 528
 529 static uint8_t
 530 RtlEqualUnicodeString(unicode_string *str1, unicode_string *str2,
 531         uint8_t caseinsensitive)
 532 {
 533         int                     i;
 534
 535         if (str1->us_len != str2->us_len)
 536                 return (FALSE);
 537
 538         for (i = 0; i < str1->us_len; i++) {
 539                 if (caseinsensitive == TRUE) {
 540                         if (toupper((char)(str1->us_buf[i] & 0xFF)) !=
 541                             toupper((char)(str2->us_buf[i] & 0xFF)))
 542                                 return (FALSE);
 543                 } else {
 544                         if (str1->us_buf[i] != str2->us_buf[i])
 545                                 return (FALSE);
 546                 }
 547         }
 548
 549         return (TRUE);
 550 }
 551
 552 static void
 553 RtlCopyString(dst, src)
 554         ansi_string             *dst;
 555         const ansi_string       *src;
 556 {
 557         if (src != NULL && src->as_buf != NULL && dst->as_buf != NULL) {
 558                 dst->as_len = min(src->as_len, dst->as_maxlen);
 559                 memcpy(dst->as_buf, src->as_buf, dst->as_len);
 560                 if (dst->as_len < dst->as_maxlen)
 561                         dst->as_buf[dst->as_len] = 0;
 562         } else
 563                 dst->as_len = 0;
 564 }
 565
 566 static void
 567 RtlCopyUnicodeString(dest, src)
 568         unicode_string          *dest;
 569         unicode_string          *src;
 570 {
 571
 572         if (dest->us_maxlen >= src->us_len)
 573                 dest->us_len = src->us_len;
 574         else
 575                 dest->us_len = dest->us_maxlen;
 576         memcpy(dest->us_buf, src->us_buf, dest->us_len);
 577 }
 578
 579 static void
 580 ntoskrnl_ascii_to_unicode(ascii, unicode, len)
 581         char                    *ascii;
 582         uint16_t                *unicode;
 583         int                     len;
 584 {
 585         int                     i;
 586         uint16_t                *ustr;
 587
 588         ustr = unicode;
 589         for (i = 0; i < len; i++) {
 590                 *ustr = (uint16_t)ascii[i];
 591                 ustr++;
 592         }
 593 }
 594
 595 static void
 596 ntoskrnl_unicode_to_ascii(unicode, ascii, len)
 597         uint16_t                *unicode;
 598         char                    *ascii;
 599         int                     len;
 600 {
 601         int                     i;
 602         uint8_t                 *astr;
 603
 604         astr = ascii;
 605         for (i = 0; i < len / 2; i++) {
 606                 *astr = (uint8_t)unicode[i];
 607                 astr++;
 608         }
 609 }
 610
 611 uint32_t
 612 RtlUnicodeStringToAnsiString(ansi_string *dest, unicode_string *src, uint8_t allocate)
 613 {
 614         if (dest == NULL || src == NULL)
 615                 return (STATUS_INVALID_PARAMETER);
 616
 617         dest->as_len = src->us_len / 2;
 618         if (dest->as_maxlen < dest->as_len)
 619                 dest->as_len = dest->as_maxlen;
 620
 621         if (allocate == TRUE) {
 622                 dest->as_buf = ExAllocatePoolWithTag(NonPagedPool,
 623                     (src->us_len / 2) + 1, 0);
 624                 if (dest->as_buf == NULL)
 625                         return (STATUS_INSUFFICIENT_RESOURCES);
 626                 dest->as_len = dest->as_maxlen = src->us_len / 2;
 627         } else {
 628                 dest->as_len = src->us_len / 2; /* XXX */
 629                 if (dest->as_maxlen < dest->as_len)
 630                         dest->as_len = dest->as_maxlen;
 631         }
 632
 633         ntoskrnl_unicode_to_ascii(src->us_buf, dest->as_buf,
 634             dest->as_len * 2);
 635
 636         return (STATUS_SUCCESS);
 637 }
 638
 639 uint32_t
 640 RtlAnsiStringToUnicodeString(unicode_string *dest, ansi_string *src,
 641         uint8_t allocate)
 642 {
 643         if (dest == NULL || src == NULL)
 644                 return (STATUS_INVALID_PARAMETER);
 645
 646         if (allocate == TRUE) {
 647                 dest->us_buf = ExAllocatePoolWithTag(NonPagedPool,
 648                     src->as_len * 2, 0);
 649                 if (dest->us_buf == NULL)
 650                         return (STATUS_INSUFFICIENT_RESOURCES);
 651                 dest->us_len = dest->us_maxlen = strlen(src->as_buf) * 2;
 652         } else {
 653                 dest->us_len = src->as_len * 2; /* XXX */
 654                 if (dest->us_maxlen < dest->us_len)
 655                         dest->us_len = dest->us_maxlen;
 656         }
 657
 658         ntoskrnl_ascii_to_unicode(src->as_buf, dest->us_buf,
 659             dest->us_len / 2);
 660
 661         return (STATUS_SUCCESS);
 662 }
 663
 664 void *
 665 ExAllocatePoolWithTag(pooltype, len, tag)
 666         uint32_t                pooltype;
 667         size_t                  len;
 668         uint32_t                tag;
 669 {
 670         void                    *buf;
 671
 672         buf = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO);
 673         if (buf == NULL)
 674                 return (NULL);
 675
 676         return (buf);
 677 }
 678
 679 static void
 680 ExFreePoolWithTag(buf, tag)
 681         void            *buf;
 682         uint32_t        tag;
 683 {
 684         ExFreePool(buf);
 685 }
 686
 687 void
 688 ExFreePool(buf)
 689         void                    *buf;
 690 {
 691         free(buf, M_DEVBUF);
 692 }
 693
 694 uint32_t
 695 IoAllocateDriverObjectExtension(drv, clid, extlen, ext)
 696         driver_object           *drv;
 697         void                    *clid;
 698         uint32_t                extlen;
 699         void                    **ext;
 700 {
 701         custom_extension        *ce;
 702
 703         ce = ExAllocatePoolWithTag(NonPagedPool, sizeof(custom_extension)
 704             + extlen, 0);
 705
 706         if (ce == NULL)
 707                 return (STATUS_INSUFFICIENT_RESOURCES);
 708
 709         ce->ce_clid = clid;
 710         InsertTailList((&drv->dro_driverext->dre_usrext), (&ce->ce_list));
 711
 712         *ext = (void *)(ce + 1);
 713
 714         return (STATUS_SUCCESS);
 715 }
 716
 717 void *
 718 IoGetDriverObjectExtension(drv, clid)
 719         driver_object           *drv;
 720         void                    *clid;
 721 {
 722         list_entry              *e;
 723         custom_extension        *ce;
 724
 725         /*
 726          * Sanity check. Our dummy bus drivers don't have
 727          * any driver extensions.
 728          */
 729
 730         if (drv->dro_driverext == NULL)
 731                 return (NULL);
 732
 733         e = drv->dro_driverext->dre_usrext.nle_flink;
 734         while (e != &drv->dro_driverext->dre_usrext) {
 735                 ce = (custom_extension *)e;
 736                 if (ce->ce_clid == clid)
 737                         return ((void *)(ce + 1));
 738                 e = e->nle_flink;
 739         }
 740
 741         return (NULL);
 742 }
 743
 744
 745 uint32_t
 746 IoCreateDevice(driver_object *drv, uint32_t devextlen, unicode_string *devname,
 747         uint32_t devtype, uint32_t devchars, uint8_t exclusive,
 748         device_object **newdev)
 749 {
 750         device_object           *dev;
 751
 752         dev = ExAllocatePoolWithTag(NonPagedPool, sizeof(device_object), 0);
 753         if (dev == NULL)
 754                 return (STATUS_INSUFFICIENT_RESOURCES);
 755
 756         dev->do_type = devtype;
 757         dev->do_drvobj = drv;
 758         dev->do_currirp = NULL;
 759         dev->do_flags = 0;
 760
 761         if (devextlen) {
 762                 dev->do_devext = ExAllocatePoolWithTag(NonPagedPool,
 763                     devextlen, 0);
 764
 765                 if (dev->do_devext == NULL) {
 766                         ExFreePool(dev);
 767                         return (STATUS_INSUFFICIENT_RESOURCES);
 768                 }
 769
 770                 bzero(dev->do_devext, devextlen);
 771         } else
 772                 dev->do_devext = NULL;
 773
 774         dev->do_size = sizeof(device_object) + devextlen;
 775         dev->do_refcnt = 1;
 776         dev->do_attacheddev = NULL;
 777         dev->do_nextdev = NULL;
 778         dev->do_devtype = devtype;
 779         dev->do_stacksize = 1;
 780         dev->do_alignreq = 1;
 781         dev->do_characteristics = devchars;
 782         dev->do_iotimer = NULL;
 783         KeInitializeEvent(&dev->do_devlock, EVENT_TYPE_SYNC, TRUE);
 784
 785         /*
 786          * Vpd is used for disk/tape devices,
 787          * but we don't support those. (Yet.)
 788          */
 789         dev->do_vpb = NULL;
 790
 791         dev->do_devobj_ext = ExAllocatePoolWithTag(NonPagedPool,
 792             sizeof(devobj_extension), 0);
 793
 794         if (dev->do_devobj_ext == NULL) {
 795                 if (dev->do_devext != NULL)
 796                         ExFreePool(dev->do_devext);
 797                 ExFreePool(dev);
 798                 return (STATUS_INSUFFICIENT_RESOURCES);
 799         }
 800
 801         dev->do_devobj_ext->dve_type = 0;
 802         dev->do_devobj_ext->dve_size = sizeof(devobj_extension);
 803         dev->do_devobj_ext->dve_devobj = dev;
 804
 805         /*
 806          * Attach this device to the driver object's list
 807          * of devices. Note: this is not the same as attaching
 808          * the device to the device stack. The driver's AddDevice
 809          * routine must explicitly call IoAddDeviceToDeviceStack()
 810          * to do that.
 811          */
 812
 813         if (drv->dro_devobj == NULL) {
 814                 drv->dro_devobj = dev;
 815                 dev->do_nextdev = NULL;
 816         } else {
 817                 dev->do_nextdev = drv->dro_devobj;
 818                 drv->dro_devobj = dev;
 819         }
 820
 821         *newdev = dev;
 822
 823         return (STATUS_SUCCESS);
 824 }
 825
 826 void
 827 IoDeleteDevice(dev)
 828         device_object           *dev;
 829 {
 830         device_object           *prev;
 831
 832         if (dev == NULL)
 833                 return;
 834
 835         if (dev->do_devobj_ext != NULL)
 836                 ExFreePool(dev->do_devobj_ext);
 837
 838         if (dev->do_devext != NULL)
 839                 ExFreePool(dev->do_devext);
 840
 841         /* Unlink the device from the driver's device list. */
 842
 843         prev = dev->do_drvobj->dro_devobj;
 844         if (prev == dev)
 845                 dev->do_drvobj->dro_devobj = dev->do_nextdev;
 846         else {
 847                 while (prev->do_nextdev != dev)
 848                         prev = prev->do_nextdev;
 849                 prev->do_nextdev = dev->do_nextdev;
 850         }
 851
 852         ExFreePool(dev);
 853 }
 854
 855 device_object *
 856 IoGetAttachedDevice(dev)
 857         device_object           *dev;
 858 {
 859         device_object           *d;
 860
 861         if (dev == NULL)
 862                 return (NULL);
 863
 864         d = dev;
 865
 866         while (d->do_attacheddev != NULL)
 867                 d = d->do_attacheddev;
 868
 869         return (d);
 870 }
 871
 872 static irp *
 873 IoBuildSynchronousFsdRequest(func, dobj, buf, len, off, event, status)
 874         uint32_t                func;
 875         device_object           *dobj;
 876         void                    *buf;
 877         uint32_t                len;
 878         uint64_t                *off;
 879         nt_kevent               *event;
 880         io_status_block         *status;
 881 {
 882         irp                     *ip;
 883
 884         ip = IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status);
 885         if (ip == NULL)
 886                 return (NULL);
 887         ip->irp_usrevent = event;
 888
 889         return (ip);
 890 }
 891
 892 static irp *
 893 IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status)
 894         uint32_t                func;
 895         device_object           *dobj;
 896         void                    *buf;
 897         uint32_t                len;
 898         uint64_t                *off;
 899         io_status_block         *status;
 900 {
 901         irp                     *ip;
 902         io_stack_location       *sl;
 903
 904         ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
 905         if (ip == NULL)
 906                 return (NULL);
 907
 908         ip->irp_usriostat = status;
 909         ip->irp_tail.irp_overlay.irp_thread = NULL;
 910
 911         sl = IoGetNextIrpStackLocation(ip);
 912         sl->isl_major = func;
 913         sl->isl_minor = 0;
 914         sl->isl_flags = 0;
 915         sl->isl_ctl = 0;
 916         sl->isl_devobj = dobj;
 917         sl->isl_fileobj = NULL;
 918         sl->isl_completionfunc = NULL;
 919
 920         ip->irp_userbuf = buf;
 921
 922         if (dobj->do_flags & DO_BUFFERED_IO) {
 923                 ip->irp_assoc.irp_sysbuf =
 924                     ExAllocatePoolWithTag(NonPagedPool, len, 0);
 925                 if (ip->irp_assoc.irp_sysbuf == NULL) {
 926                         IoFreeIrp(ip);
 927                         return (NULL);
 928                 }
 929                 bcopy(buf, ip->irp_assoc.irp_sysbuf, len);
 930         }
 931
 932         if (dobj->do_flags & DO_DIRECT_IO) {
 933                 ip->irp_mdl = IoAllocateMdl(buf, len, FALSE, FALSE, ip);
 934                 if (ip->irp_mdl == NULL) {
 935                         if (ip->irp_assoc.irp_sysbuf != NULL)
 936                                 ExFreePool(ip->irp_assoc.irp_sysbuf);
 937                         IoFreeIrp(ip);
 938                         return (NULL);
 939                 }
 940                 ip->irp_userbuf = NULL;
 941                 ip->irp_assoc.irp_sysbuf = NULL;
 942         }
 943
 944         if (func == IRP_MJ_READ) {
 945                 sl->isl_parameters.isl_read.isl_len = len;
 946                 if (off != NULL)
 947                         sl->isl_parameters.isl_read.isl_byteoff = *off;
 948                 else
 949                         sl->isl_parameters.isl_read.isl_byteoff = 0;
 950         }
 951
 952         if (func == IRP_MJ_WRITE) {
 953                 sl->isl_parameters.isl_write.isl_len = len;
 954                 if (off != NULL)
 955                         sl->isl_parameters.isl_write.isl_byteoff = *off;
 956                 else
 957                         sl->isl_parameters.isl_write.isl_byteoff = 0;
 958         }
 959
 960         return (ip);
 961 }
 962
 963 static irp *
 964 IoBuildDeviceIoControlRequest(uint32_t iocode, device_object *dobj, void *ibuf,
 965         uint32_t ilen, void *obuf, uint32_t olen, uint8_t isinternal,
 966         nt_kevent *event, io_status_block *status)
 967 {
 968         irp                     *ip;
 969         io_stack_location       *sl;
 970         uint32_t                buflen;
 971
 972         ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
 973         if (ip == NULL)
 974                 return (NULL);
 975         ip->irp_usrevent = event;
 976         ip->irp_usriostat = status;
 977         ip->irp_tail.irp_overlay.irp_thread = NULL;
 978
 979         sl = IoGetNextIrpStackLocation(ip);
 980         sl->isl_major = isinternal == TRUE ?
 981             IRP_MJ_INTERNAL_DEVICE_CONTROL : IRP_MJ_DEVICE_CONTROL;
 982         sl->isl_minor = 0;
 983         sl->isl_flags = 0;
 984         sl->isl_ctl = 0;
 985         sl->isl_devobj = dobj;
 986         sl->isl_fileobj = NULL;
 987         sl->isl_completionfunc = NULL;
 988         sl->isl_parameters.isl_ioctl.isl_iocode = iocode;
 989         sl->isl_parameters.isl_ioctl.isl_ibuflen = ilen;
 990         sl->isl_parameters.isl_ioctl.isl_obuflen = olen;
 991
 992         switch(IO_METHOD(iocode)) {
 993         case METHOD_BUFFERED:
 994                 if (ilen > olen)
 995                         buflen = ilen;
 996                 else
 997                         buflen = olen;
 998                 if (buflen) {
 999                         ip->irp_assoc.irp_sysbuf =
1000                             ExAllocatePoolWithTag(NonPagedPool, buflen, 0);
1001                         if (ip->irp_assoc.irp_sysbuf == NULL) {
1002                                 IoFreeIrp(ip);
1003                                 return (NULL);
1004                         }
1005                 }
1006                 if (ilen && ibuf != NULL) {
1007                         bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1008                         bzero((char *)ip->irp_assoc.irp_sysbuf + ilen,
1009                             buflen - ilen);
1010                 } else
1011                         bzero(ip->irp_assoc.irp_sysbuf, ilen);
1012                 ip->irp_userbuf = obuf;
1013                 break;
1014         case METHOD_IN_DIRECT:
1015         case METHOD_OUT_DIRECT:
1016                 if (ilen && ibuf != NULL) {
1017                         ip->irp_assoc.irp_sysbuf =
1018                             ExAllocatePoolWithTag(NonPagedPool, ilen, 0);
1019                         if (ip->irp_assoc.irp_sysbuf == NULL) {
1020                                 IoFreeIrp(ip);
1021                                 return (NULL);
1022                         }
1023                         bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1024                 }
1025                 if (olen && obuf != NULL) {
1026                         ip->irp_mdl = IoAllocateMdl(obuf, olen,
1027                             FALSE, FALSE, ip);
1028                         /*
1029                          * Normally we would MmProbeAndLockPages()
1030                          * here, but we don't have to in our
1031                          * imlementation.
1032                          */
1033                 }
1034                 break;
1035         case METHOD_NEITHER:
1036                 ip->irp_userbuf = obuf;
1037                 sl->isl_parameters.isl_ioctl.isl_type3ibuf = ibuf;
1038                 break;
1039         default:
1040                 break;
1041         }
1042
1043         /*
1044          * Ideally, we should associate this IRP with the calling
1045          * thread here.
1046          */
1047
1048         return (ip);
1049 }
1050
1051 static irp *
1052 IoAllocateIrp(uint8_t stsize, uint8_t chargequota)
1053 {
1054         irp                     *i;
1055
1056         i = ExAllocatePoolWithTag(NonPagedPool, IoSizeOfIrp(stsize), 0);
1057         if (i == NULL)
1058                 return (NULL);
1059
1060         IoInitializeIrp(i, IoSizeOfIrp(stsize), stsize);
1061
1062         return (i);
1063 }
1064
1065 static irp *
1066 IoMakeAssociatedIrp(irp *ip, uint8_t stsize)
1067 {
1068         irp                     *associrp;
1069
1070         associrp = IoAllocateIrp(stsize, FALSE);
1071         if (associrp == NULL)
1072                 return (NULL);
1073
1074         mtx_lock(&ntoskrnl_dispatchlock);
1075         associrp->irp_flags |= IRP_ASSOCIATED_IRP;
1076         associrp->irp_tail.irp_overlay.irp_thread =
1077             ip->irp_tail.irp_overlay.irp_thread;
1078         associrp->irp_assoc.irp_master = ip;
1079         mtx_unlock(&ntoskrnl_dispatchlock);
1080
1081         return (associrp);
1082 }
1083
1084 static void
1085 IoFreeIrp(ip)
1086         irp                     *ip;
1087 {
1088         ExFreePool(ip);
1089 }
1090
1091 static void
1092 IoInitializeIrp(irp *io, uint16_t psize, uint8_t ssize)
1093 {
1094         bzero((char *)io, IoSizeOfIrp(ssize));
1095         io->irp_size = psize;
1096         io->irp_stackcnt = ssize;
1097         io->irp_currentstackloc = ssize;
1098         InitializeListHead(&io->irp_thlist);
1099         io->irp_tail.irp_overlay.irp_csl =
1100             (io_stack_location *)(io + 1) + ssize;
1101 }
1102
1103 static void
1104 IoReuseIrp(ip, status)
1105         irp                     *ip;
1106         uint32_t                status;
1107 {
1108         uint8_t                 allocflags;
1109
1110         allocflags = ip->irp_allocflags;
1111         IoInitializeIrp(ip, ip->irp_size, ip->irp_stackcnt);
1112         ip->irp_iostat.isb_status = status;
1113         ip->irp_allocflags = allocflags;
1114 }
1115
1116 void
1117 IoAcquireCancelSpinLock(uint8_t *irql)
1118 {
1119         KeAcquireSpinLock(&ntoskrnl_cancellock, irql);
1120 }
1121
1122 void
1123 IoReleaseCancelSpinLock(uint8_t irql)
1124 {
1125         KeReleaseSpinLock(&ntoskrnl_cancellock, irql);
1126 }
1127
1128 uint8_t
1129 IoCancelIrp(irp *ip)
1130 {
1131         cancel_func             cfunc;
1132         uint8_t                 cancelirql;
1133
1134         IoAcquireCancelSpinLock(&cancelirql);
1135         cfunc = IoSetCancelRoutine(ip, NULL);
1136         ip->irp_cancel = TRUE;
1137         if (cfunc == NULL) {
1138                 IoReleaseCancelSpinLock(cancelirql);
1139                 return (FALSE);
1140         }
1141         ip->irp_cancelirql = cancelirql;
1142         MSCALL2(cfunc, IoGetCurrentIrpStackLocation(ip)->isl_devobj, ip);
1143         return (uint8_t)IoSetCancelValue(ip, TRUE);
1144 }
1145
1146 uint32_t
1147 IofCallDriver(dobj, ip)
1148         device_object           *dobj;
1149         irp                     *ip;
1150 {
1151         driver_object           *drvobj;
1152         io_stack_location       *sl;
1153         uint32_t                status;
1154         driver_dispatch         disp;
1155
1156         drvobj = dobj->do_drvobj;
1157
1158         if (ip->irp_currentstackloc <= 0)
1159                 panic("IoCallDriver(): out of stack locations");
1160
1161         IoSetNextIrpStackLocation(ip);
1162         sl = IoGetCurrentIrpStackLocation(ip);
1163
1164         sl->isl_devobj = dobj;
1165
1166         disp = drvobj->dro_dispatch[sl->isl_major];
1167         status = MSCALL2(disp, dobj, ip);
1168
1169         return (status);
1170 }
1171
1172 void
1173 IofCompleteRequest(irp *ip, uint8_t prioboost)
1174 {
1175         uint32_t                status;
1176         device_object           *dobj;
1177         io_stack_location       *sl;
1178         completion_func         cf;
1179
1180         KASSERT(ip->irp_iostat.isb_status != STATUS_PENDING,
1181             ("incorrect IRP(%p) status (STATUS_PENDING)", ip));
1182
1183         sl = IoGetCurrentIrpStackLocation(ip);
1184         IoSkipCurrentIrpStackLocation(ip);
1185
1186         do {
1187                 if (sl->isl_ctl & SL_PENDING_RETURNED)
1188                         ip->irp_pendingreturned = TRUE;
1189
1190                 if (ip->irp_currentstackloc != (ip->irp_stackcnt + 1))
1191                         dobj = IoGetCurrentIrpStackLocation(ip)->isl_devobj;
1192                 else
1193                         dobj = NULL;
1194
1195                 if (sl->isl_completionfunc != NULL &&
1196                     ((ip->irp_iostat.isb_status == STATUS_SUCCESS &&
1197                     sl->isl_ctl & SL_INVOKE_ON_SUCCESS) ||
1198                     (ip->irp_iostat.isb_status != STATUS_SUCCESS &&
1199                     sl->isl_ctl & SL_INVOKE_ON_ERROR) ||
1200                     (ip->irp_cancel == TRUE &&
1201                     sl->isl_ctl & SL_INVOKE_ON_CANCEL))) {
1202                         cf = sl->isl_completionfunc;
1203                         status = MSCALL3(cf, dobj, ip, sl->isl_completionctx);
1204                         if (status == STATUS_MORE_PROCESSING_REQUIRED)
1205                                 return;
1206                 } else {
1207                         if ((ip->irp_currentstackloc <= ip->irp_stackcnt) &&
1208                             (ip->irp_pendingreturned == TRUE))
1209                                 IoMarkIrpPending(ip);
1210                 }
1211
1212                 /* move to the next.  */
1213                 IoSkipCurrentIrpStackLocation(ip);
1214                 sl++;
1215         } while (ip->irp_currentstackloc <= (ip->irp_stackcnt + 1));
1216
1217         if (ip->irp_usriostat != NULL)
1218                 *ip->irp_usriostat = ip->irp_iostat;
1219         if (ip->irp_usrevent != NULL)
1220                 KeSetEvent(ip->irp_usrevent, prioboost, FALSE);
1221
1222         /* Handle any associated IRPs. */
1223
1224         if (ip->irp_flags & IRP_ASSOCIATED_IRP) {
1225                 uint32_t                masterirpcnt;
1226                 irp                     *masterirp;
1227                 mdl                     *m;
1228
1229                 masterirp = ip->irp_assoc.irp_master;
1230                 masterirpcnt =
1231                     InterlockedDecrement(&masterirp->irp_assoc.irp_irpcnt);
1232
1233                 while ((m = ip->irp_mdl) != NULL) {
1234                         ip->irp_mdl = m->mdl_next;
1235                         IoFreeMdl(m);
1236                 }
1237                 IoFreeIrp(ip);
1238                 if (masterirpcnt == 0)
1239                         IoCompleteRequest(masterirp, IO_NO_INCREMENT);
1240                 return;
1241         }
1242
1243         /* With any luck, these conditions will never arise. */
1244
1245         if (ip->irp_flags & IRP_PAGING_IO) {
1246                 if (ip->irp_mdl != NULL)
1247                         IoFreeMdl(ip->irp_mdl);
1248                 IoFreeIrp(ip);
1249         }
1250 }
1251
1252 void
1253 ntoskrnl_intr(arg)
1254         void                    *arg;
1255 {
1256         kinterrupt              *iobj;
1257         uint8_t                 irql;
1258         uint8_t                 claimed;
1259         list_entry              *l;
1260
1261         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1262         l = ntoskrnl_intlist.nle_flink;
1263         while (l != &ntoskrnl_intlist) {
1264                 iobj = CONTAINING_RECORD(l, kinterrupt, ki_list);
1265                 claimed = MSCALL2(iobj->ki_svcfunc, iobj, iobj->ki_svcctx);
1266                 if (claimed == TRUE)
1267                         break;
1268                 l = l->nle_flink;
1269         }
1270         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1271 }
1272
1273 uint8_t
1274 KeAcquireInterruptSpinLock(iobj)
1275         kinterrupt              *iobj;
1276 {
1277         uint8_t                 irql;
1278         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1279         return (irql);
1280 }
1281
1282 void
1283 KeReleaseInterruptSpinLock(kinterrupt *iobj, uint8_t irql)
1284 {
1285         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1286 }
1287
1288 uint8_t
1289 KeSynchronizeExecution(iobj, syncfunc, syncctx)
1290         kinterrupt              *iobj;
1291         void                    *syncfunc;
1292         void                    *syncctx;
1293 {
1294         uint8_t                 irql;
1295
1296         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1297         MSCALL1(syncfunc, syncctx);
1298         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1299
1300         return (TRUE);
1301 }
1302
1303 /*
1304  * IoConnectInterrupt() is passed only the interrupt vector and
1305  * irql that a device wants to use, but no device-specific tag
1306  * of any kind. This conflicts rather badly with FreeBSD's
1307  * bus_setup_intr(), which needs the device_t for the device
1308  * requesting interrupt delivery. In order to bypass this
1309  * inconsistency, we implement a second level of interrupt
1310  * dispatching on top of bus_setup_intr(). All devices use
1311  * ntoskrnl_intr() as their ISR, and any device requesting
1312  * interrupts will be registered with ntoskrnl_intr()'s interrupt
1313  * dispatch list. When an interrupt arrives, we walk the list
1314  * and invoke all the registered ISRs. This effectively makes all
1315  * interrupts shared, but it's the only way to duplicate the
1316  * semantics of IoConnectInterrupt() and IoDisconnectInterrupt() properly.
1317  */
1318
1319 uint32_t
1320 IoConnectInterrupt(kinterrupt **iobj, void *svcfunc, void *svcctx,
1321         kspin_lock *lock, uint32_t vector, uint8_t irql, uint8_t syncirql,
1322         uint8_t imode, uint8_t shared, uint32_t affinity, uint8_t savefloat)
1323 {
1324         uint8_t                 curirql;
1325
1326         *iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0);
1327         if (*iobj == NULL)
1328                 return (STATUS_INSUFFICIENT_RESOURCES);
1329
1330         (*iobj)->ki_svcfunc = svcfunc;
1331         (*iobj)->ki_svcctx = svcctx;
1332
1333         if (lock == NULL) {
1334                 KeInitializeSpinLock(&(*iobj)->ki_lock_priv);
1335                 (*iobj)->ki_lock = &(*iobj)->ki_lock_priv;
1336         } else
1337                 (*iobj)->ki_lock = lock;
1338
1339         KeAcquireSpinLock(&ntoskrnl_intlock, &curirql);
1340         InsertHeadList((&ntoskrnl_intlist), (&(*iobj)->ki_list));
1341         KeReleaseSpinLock(&ntoskrnl_intlock, curirql);
1342
1343         return (STATUS_SUCCESS);
1344 }
1345
1346 void
1347 IoDisconnectInterrupt(iobj)
1348         kinterrupt              *iobj;
1349 {
1350         uint8_t                 irql;
1351
1352         if (iobj == NULL)
1353                 return;
1354
1355         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1356         RemoveEntryList((&iobj->ki_list));
1357         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1358
1359         ExFreePool(iobj);
1360 }
1361
1362 device_object *
1363 IoAttachDeviceToDeviceStack(src, dst)
1364         device_object           *src;
1365         device_object           *dst;
1366 {
1367         device_object           *attached;
1368
1369         mtx_lock(&ntoskrnl_dispatchlock);
1370         attached = IoGetAttachedDevice(dst);
1371         attached->do_attacheddev = src;
1372         src->do_attacheddev = NULL;
1373         src->do_stacksize = attached->do_stacksize + 1;
1374         mtx_unlock(&ntoskrnl_dispatchlock);
1375
1376         return (attached);
1377 }
1378
1379 void
1380 IoDetachDevice(topdev)
1381         device_object           *topdev;
1382 {
1383         device_object           *tail;
1384
1385         mtx_lock(&ntoskrnl_dispatchlock);
1386
1387         /* First, break the chain. */
1388         tail = topdev->do_attacheddev;
1389         if (tail == NULL) {
1390                 mtx_unlock(&ntoskrnl_dispatchlock);
1391                 return;
1392         }
1393         topdev->do_attacheddev = tail->do_attacheddev;
1394         topdev->do_refcnt--;
1395
1396         /* Now reduce the stacksize count for the takm_il objects. */
1397
1398         tail = topdev->do_attacheddev;
1399         while (tail != NULL) {
1400                 tail->do_stacksize--;
1401                 tail = tail->do_attacheddev;
1402         }
1403
1404         mtx_unlock(&ntoskrnl_dispatchlock);
1405 }
1406
1407 /*
1408  * For the most part, an object is considered signalled if
1409  * dh_sigstate == TRUE. The exception is for mutant objects
1410  * (mutexes), where the logic works like this:
1411  *
1412  * - If the thread already owns the object and sigstate is
1413  *   less than or equal to 0, then the object is considered
1414  *   signalled (recursive acquisition).
1415  * - If dh_sigstate == 1, the object is also considered
1416  *   signalled.
1417  */
1418
1419 static int
1420 ntoskrnl_is_signalled(obj, td)
1421         nt_dispatch_header      *obj;
1422         struct thread           *td;
1423 {
1424         kmutant                 *km;
1425
1426         if (obj->dh_type == DISP_TYPE_MUTANT) {
1427                 km = (kmutant *)obj;
1428                 if ((obj->dh_sigstate <= 0 && km->km_ownerthread == td) ||
1429                     obj->dh_sigstate == 1)
1430                         return (TRUE);
1431                 return (FALSE);
1432         }
1433
1434         if (obj->dh_sigstate > 0)
1435                 return (TRUE);
1436         return (FALSE);
1437 }
1438
1439 static void
1440 ntoskrnl_satisfy_wait(obj, td)
1441         nt_dispatch_header      *obj;
1442         struct thread           *td;
1443 {
1444         kmutant                 *km;
1445
1446         switch (obj->dh_type) {
1447         case DISP_TYPE_MUTANT:
1448                 km = (struct kmutant *)obj;
1449                 obj->dh_sigstate--;
1450                 /*
1451                  * If sigstate reaches 0, the mutex is now
1452                  * non-signalled (the new thread owns it).
1453                  */
1454                 if (obj->dh_sigstate == 0) {
1455                         km->km_ownerthread = td;
1456                         if (km->km_abandoned == TRUE)
1457                                 km->km_abandoned = FALSE;
1458                 }
1459                 break;
1460         /* Synchronization objects get reset to unsignalled. */
1461         case DISP_TYPE_SYNCHRONIZATION_EVENT:
1462         case DISP_TYPE_SYNCHRONIZATION_TIMER:
1463                 obj->dh_sigstate = 0;
1464                 break;
1465         case DISP_TYPE_SEMAPHORE:
1466                 obj->dh_sigstate--;
1467                 break;
1468         default:
1469                 break;
1470         }
1471 }
1472
1473 static void
1474 ntoskrnl_satisfy_multiple_waits(wb)
1475         wait_block              *wb;
1476 {
1477         wait_block              *cur;
1478         struct thread           *td;
1479
1480         cur = wb;
1481         td = wb->wb_kthread;
1482
1483         do {
1484                 ntoskrnl_satisfy_wait(wb->wb_object, td);
1485                 cur->wb_awakened = TRUE;
1486                 cur = cur->wb_next;
1487         } while (cur != wb);
1488 }
1489
1490 /* Always called with dispatcher lock held. */
1491 static void
1492 ntoskrnl_waittest(obj, increment)
1493         nt_dispatch_header      *obj;
1494         uint32_t                increment;
1495 {
1496         wait_block              *w, *next;
1497         list_entry              *e;
1498         struct thread           *td;
1499         wb_ext                  *we;
1500         int                     satisfied;
1501
1502         /*
1503          * Once an object has been signalled, we walk its list of
1504          * wait blocks. If a wait block can be awakened, then satisfy
1505          * waits as necessary and wake the thread.
1506          *
1507          * The rules work like this:
1508          *
1509          * If a wait block is marked as WAITTYPE_ANY, then
1510          * we can satisfy the wait conditions on the current
1511          * object and wake the thread right away. Satisfying
1512          * the wait also has the effect of breaking us out
1513          * of the search loop.
1514          *
1515          * If the object is marked as WAITTYLE_ALL, then the
1516          * wait block will be part of a circularly linked
1517          * list of wait blocks belonging to a waiting thread
1518          * that's sleeping in KeWaitForMultipleObjects(). In
1519          * order to wake the thread, all the objects in the
1520          * wait list must be in the signalled state. If they
1521          * are, we then satisfy all of them and wake the
1522          * thread.
1523          *
1524          */
1525
1526         e = obj->dh_waitlisthead.nle_flink;
1527
1528         while (e != &obj->dh_waitlisthead && obj->dh_sigstate > 0) {
1529                 w = CONTAINING_RECORD(e, wait_block, wb_waitlist);
1530                 we = w->wb_ext;
1531                 td = we->we_td;
1532                 satisfied = FALSE;
1533                 if (w->wb_waittype == WAITTYPE_ANY) {
1534                         /*
1535                          * Thread can be awakened if
1536                          * any wait is satisfied.
1537                          */
1538                         ntoskrnl_satisfy_wait(obj, td);
1539                         satisfied = TRUE;
1540                         w->wb_awakened = TRUE;
1541                 } else {
1542                         /*
1543                          * Thread can only be woken up
1544                          * if all waits are satisfied.
1545                          * If the thread is waiting on multiple
1546                          * objects, they should all be linked
1547                          * through the wb_next pointers in the
1548                          * wait blocks.
1549                          */
1550                         satisfied = TRUE;
1551                         next = w->wb_next;
1552                         while (next != w) {
1553                                 if (ntoskrnl_is_signalled(obj, td) == FALSE) {
1554                                         satisfied = FALSE;
1555                                         break;
1556                                 }
1557                                 next = next->wb_next;
1558                         }
1559                         ntoskrnl_satisfy_multiple_waits(w);
1560                 }
1561
1562                 if (satisfied == TRUE)
1563                         cv_broadcastpri(&we->we_cv,
1564                             (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
1565                             w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
1566
1567                 e = e->nle_flink;
1568         }
1569 }
1570
1571 /*
1572  * Return the number of 100 nanosecond intervals since
1573  * January 1, 1601. (?!?!)
1574  */
1575 void
1576 ntoskrnl_time(tval)
1577         uint64_t                *tval;
1578 {
1579         struct timespec         ts;
1580
1581         nanotime(&ts);
1582         *tval = (uint64_t)ts.tv_nsec / 100 + (uint64_t)ts.tv_sec * 10000000 +
1583             11644473600 * 10000000; /* 100ns ticks from 1601 to 1970 */
1584 }
1585
1586 static void
1587 KeQuerySystemTime(current_time)
1588         uint64_t                *current_time;
1589 {
1590         ntoskrnl_time(current_time);
1591 }
1592
1593 static uint32_t
1594 KeTickCount(void)
1595 {
1596         struct timeval tv;
1597         getmicrouptime(&tv);
1598         return tvtohz(&tv);
1599 }
1600
1601
1602 /*
1603  * KeWaitForSingleObject() is a tricky beast, because it can be used
1604  * with several different object types: semaphores, timers, events,
1605  * mutexes and threads. Semaphores don't appear very often, but the
1606  * other object types are quite common. KeWaitForSingleObject() is
1607  * what's normally used to acquire a mutex, and it can be used to
1608  * wait for a thread termination.
1609  *
1610  * The Windows NDIS API is implemented in terms of Windows kernel
1611  * primitives, and some of the object manipulation is duplicated in
1612  * NDIS. For example, NDIS has timers and events, which are actually
1613  * Windows kevents and ktimers. Now, you're supposed to only use the
1614  * NDIS variants of these objects within the confines of the NDIS API,
1615  * but there are some naughty developers out there who will use
1616  * KeWaitForSingleObject() on NDIS timer and event objects, so we
1617  * have to support that as well. Conseqently, our NDIS timer and event
1618  * code has to be closely tied into our ntoskrnl timer and event code,
1619  * just as it is in Windows.
1620  *
1621  * KeWaitForSingleObject() may do different things for different kinds
1622  * of objects:
1623  *
1624  * - For events, we check if the event has been signalled. If the
1625  *   event is already in the signalled state, we just return immediately,
1626  *   otherwise we wait for it to be set to the signalled state by someone
1627  *   else calling KeSetEvent(). Events can be either synchronization or
1628  *   notification events.
1629  *
1630  * - For timers, if the timer has already fired and the timer is in
1631  *   the signalled state, we just return, otherwise we wait on the
1632  *   timer. Unlike an event, timers get signalled automatically when
1633  *   they expire rather than someone having to trip them manually.
1634  *   Timers initialized with KeInitializeTimer() are always notification
1635  *   events: KeInitializeTimerEx() lets you initialize a timer as
1636  *   either a notification or synchronization event.
1637  *
1638  * - For mutexes, we try to acquire the mutex and if we can't, we wait
1639  *   on the mutex until it's available and then grab it. When a mutex is
1640  *   released, it enters the signalled state, which wakes up one of the
1641  *   threads waiting to acquire it. Mutexes are always synchronization
1642  *   events.
1643  *
1644  * - For threads, the only thing we do is wait until the thread object
1645  *   enters a signalled state, which occurs when the thread terminates.
1646  *   Threads are always notification events.
1647  *
1648  * A notification event wakes up all threads waiting on an object. A
1649  * synchronization event wakes up just one. Also, a synchronization event
1650  * is auto-clearing, which means we automatically set the event back to
1651  * the non-signalled state once the wakeup is done.
1652  */
1653
1654 uint32_t
1655 KeWaitForSingleObject(void *arg, uint32_t reason, uint32_t mode,
1656     uint8_t alertable, int64_t *duetime)
1657 {
1658         wait_block              w;
1659         struct thread           *td = curthread;
1660         struct timeval          tv;
1661         int                     error = 0;
1662         uint64_t                curtime;
1663         wb_ext                  we;
1664         nt_dispatch_header      *obj;
1665
1666         obj = arg;
1667
1668         if (obj == NULL)
1669                 return (STATUS_INVALID_PARAMETER);
1670
1671         mtx_lock(&ntoskrnl_dispatchlock);
1672
1673         cv_init(&we.we_cv, "KeWFS");
1674         we.we_td = td;
1675
1676         /*
1677          * Check to see if this object is already signalled,
1678          * and just return without waiting if it is.
1679          */
1680         if (ntoskrnl_is_signalled(obj, td) == TRUE) {
1681                 /* Sanity check the signal state value. */
1682                 if (obj->dh_sigstate != INT32_MIN) {
1683                         ntoskrnl_satisfy_wait(obj, curthread);
1684                         mtx_unlock(&ntoskrnl_dispatchlock);
1685                         return (STATUS_SUCCESS);
1686                 } else {
1687                         /*
1688                          * There's a limit to how many times we can
1689                          * recursively acquire a mutant. If we hit
1690                          * the limit, something is very wrong.
1691                          */
1692                         if (obj->dh_type == DISP_TYPE_MUTANT) {
1693                                 mtx_unlock(&ntoskrnl_dispatchlock);
1694                                 panic("mutant limit exceeded");
1695                         }
1696                 }
1697         }
1698
1699         bzero((char *)&w, sizeof(wait_block));
1700         w.wb_object = obj;
1701         w.wb_ext = &we;
1702         w.wb_waittype = WAITTYPE_ANY;
1703         w.wb_next = &w;
1704         w.wb_waitkey = 0;
1705         w.wb_awakened = FALSE;
1706         w.wb_oldpri = td->td_priority;
1707
1708         InsertTailList((&obj->dh_waitlisthead), (&w.wb_waitlist));
1709
1710         /*
1711          * The timeout value is specified in 100 nanosecond units
1712          * and can be a positive or negative number. If it's positive,
1713          * then the duetime is absolute, and we need to convert it
1714          * to an absolute offset relative to now in order to use it.
1715          * If it's negative, then the duetime is relative and we
1716          * just have to convert the units.
1717          */
1718
1719         if (duetime != NULL) {
1720                 if (*duetime < 0) {
1721                         tv.tv_sec = - (*duetime) / 10000000;
1722                         tv.tv_usec = (- (*duetime) / 10) -
1723                             (tv.tv_sec * 1000000);
1724                 } else {
1725                         ntoskrnl_time(&curtime);
1726                         if (*duetime < curtime)
1727                                 tv.tv_sec = tv.tv_usec = 0;
1728                         else {
1729                                 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1730                                 tv.tv_usec = ((*duetime) - curtime) / 10 -
1731                                     (tv.tv_sec * 1000000);
1732                         }
1733                 }
1734         }
1735
1736         if (duetime == NULL)
1737                 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1738         else
1739                 error = cv_timedwait(&we.we_cv,
1740                     &ntoskrnl_dispatchlock, tvtohz(&tv));
1741
1742         RemoveEntryList(&w.wb_waitlist);
1743
1744         cv_destroy(&we.we_cv);
1745
1746         /* We timed out. Leave the object alone and return status. */
1747
1748         if (error == EWOULDBLOCK) {
1749                 mtx_unlock(&ntoskrnl_dispatchlock);
1750                 return (STATUS_TIMEOUT);
1751         }
1752
1753         mtx_unlock(&ntoskrnl_dispatchlock);
1754
1755         return (STATUS_SUCCESS);
1756 /*
1757         return (KeWaitForMultipleObjects(1, &obj, WAITTYPE_ALL, reason,
1758             mode, alertable, duetime, &w));
1759 */
1760 }
1761
1762 static uint32_t
1763 KeWaitForMultipleObjects(uint32_t cnt, nt_dispatch_header *obj[], uint32_t wtype,
1764         uint32_t reason, uint32_t mode, uint8_t alertable, int64_t *duetime,
1765         wait_block *wb_array)
1766 {
1767         struct thread           *td = curthread;
1768         wait_block              *whead, *w;
1769         wait_block              _wb_array[MAX_WAIT_OBJECTS];
1770         nt_dispatch_header      *cur;
1771         struct timeval          tv;
1772         int                     i, wcnt = 0, error = 0;
1773         uint64_t                curtime;
1774         struct timespec         t1, t2;
1775         uint32_t                status = STATUS_SUCCESS;
1776         wb_ext                  we;
1777
1778         if (cnt > MAX_WAIT_OBJECTS)
1779                 return (STATUS_INVALID_PARAMETER);
1780         if (cnt > THREAD_WAIT_OBJECTS && wb_array == NULL)
1781                 return (STATUS_INVALID_PARAMETER);
1782
1783         mtx_lock(&ntoskrnl_dispatchlock);
1784
1785         cv_init(&we.we_cv, "KeWFM");
1786         we.we_td = td;
1787
1788         if (wb_array == NULL)
1789                 whead = _wb_array;
1790         else
1791                 whead = wb_array;
1792
1793         bzero((char *)whead, sizeof(wait_block) * cnt);
1794
1795         /* First pass: see if we can satisfy any waits immediately. */
1796
1797         wcnt = 0;
1798         w = whead;
1799
1800         for (i = 0; i < cnt; i++) {
1801                 InsertTailList((&obj[i]->dh_waitlisthead),
1802                     (&w->wb_waitlist));
1803                 w->wb_ext = &we;
1804                 w->wb_object = obj[i];
1805                 w->wb_waittype = wtype;
1806                 w->wb_waitkey = i;
1807                 w->wb_awakened = FALSE;
1808                 w->wb_oldpri = td->td_priority;
1809                 w->wb_next = w + 1;
1810                 w++;
1811                 wcnt++;
1812                 if (ntoskrnl_is_signalled(obj[i], td)) {
1813                         /*
1814                          * There's a limit to how many times
1815                          * we can recursively acquire a mutant.
1816                          * If we hit the limit, something
1817                          * is very wrong.
1818                          */
1819                         if (obj[i]->dh_sigstate == INT32_MIN &&
1820                             obj[i]->dh_type == DISP_TYPE_MUTANT) {
1821                                 mtx_unlock(&ntoskrnl_dispatchlock);
1822                                 panic("mutant limit exceeded");
1823                         }
1824
1825                         /*
1826                          * If this is a WAITTYPE_ANY wait, then
1827                          * satisfy the waited object and exit
1828                          * right now.
1829                          */
1830
1831                         if (wtype == WAITTYPE_ANY) {
1832                                 ntoskrnl_satisfy_wait(obj[i], td);
1833                                 status = STATUS_WAIT_0 + i;
1834                                 goto wait_done;
1835                         } else {
1836                                 w--;
1837                                 wcnt--;
1838                                 w->wb_object = NULL;
1839                                 RemoveEntryList(&w->wb_waitlist);
1840                         }
1841                 }
1842         }
1843
1844         /*
1845          * If this is a WAITTYPE_ALL wait and all objects are
1846          * already signalled, satisfy the waits and exit now.
1847          */
1848
1849         if (wtype == WAITTYPE_ALL && wcnt == 0) {
1850                 for (i = 0; i < cnt; i++)
1851                         ntoskrnl_satisfy_wait(obj[i], td);
1852                 status = STATUS_SUCCESS;
1853                 goto wait_done;
1854         }
1855
1856         /*
1857          * Create a circular waitblock list. The waitcount
1858          * must always be non-zero when we get here.
1859          */
1860
1861         (w - 1)->wb_next = whead;
1862
1863         /* Wait on any objects that aren't yet signalled. */
1864
1865         /* Calculate timeout, if any. */
1866
1867         if (duetime != NULL) {
1868                 if (*duetime < 0) {
1869                         tv.tv_sec = - (*duetime) / 10000000;
1870                         tv.tv_usec = (- (*duetime) / 10) -
1871                             (tv.tv_sec * 1000000);
1872                 } else {
1873                         ntoskrnl_time(&curtime);
1874                         if (*duetime < curtime)
1875                                 tv.tv_sec = tv.tv_usec = 0;
1876                         else {
1877                                 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1878                                 tv.tv_usec = ((*duetime) - curtime) / 10 -
1879                                     (tv.tv_sec * 1000000);
1880                         }
1881                 }
1882         }
1883
1884         while (wcnt) {
1885                 nanotime(&t1);
1886
1887                 if (duetime == NULL)
1888                         cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1889                 else
1890                         error = cv_timedwait(&we.we_cv,
1891                             &ntoskrnl_dispatchlock, tvtohz(&tv));
1892
1893                 /* Wait with timeout expired. */
1894
1895                 if (error) {
1896                         status = STATUS_TIMEOUT;
1897                         goto wait_done;
1898                 }
1899
1900                 nanotime(&t2);
1901
1902                 /* See what's been signalled. */
1903
1904                 w = whead;
1905                 do {
1906                         cur = w->wb_object;
1907                         if (ntoskrnl_is_signalled(cur, td) == TRUE ||
1908                             w->wb_awakened == TRUE) {
1909                                 /* Sanity check the signal state value. */
1910                                 if (cur->dh_sigstate == INT32_MIN &&
1911                                     cur->dh_type == DISP_TYPE_MUTANT) {
1912                                         mtx_unlock(&ntoskrnl_dispatchlock);
1913                                         panic("mutant limit exceeded");
1914                                 }
1915                                 wcnt--;
1916                                 if (wtype == WAITTYPE_ANY) {
1917                                         status = w->wb_waitkey &
1918                                             STATUS_WAIT_0;
1919                                         goto wait_done;
1920                                 }
1921                         }
1922                         w = w->wb_next;
1923                 } while (w != whead);
1924
1925                 /*
1926                  * If all objects have been signalled, or if this
1927                  * is a WAITTYPE_ANY wait and we were woke up by
1928                  * someone, we can bail.
1929                  */
1930
1931                 if (wcnt == 0) {
1932                         status = STATUS_SUCCESS;
1933                         goto wait_done;
1934                 }
1935
1936                 /*
1937                  * If this is WAITTYPE_ALL wait, and there's still
1938                  * objects that haven't been signalled, deduct the
1939                  * time that's elapsed so far from the timeout and
1940                  * wait again (or continue waiting indefinitely if
1941                  * there's no timeout).
1942                  */
1943
1944                 if (duetime != NULL) {
1945                         tv.tv_sec -= (t2.tv_sec - t1.tv_sec);
1946                         tv.tv_usec -= (t2.tv_nsec - t1.tv_nsec) / 1000;
1947                 }
1948         }
1949
1950
1951 wait_done:
1952
1953         cv_destroy(&we.we_cv);
1954
1955         for (i = 0; i < cnt; i++) {
1956                 if (whead[i].wb_object != NULL)
1957                         RemoveEntryList(&whead[i].wb_waitlist);
1958
1959         }
1960         mtx_unlock(&ntoskrnl_dispatchlock);
1961
1962         return (status);
1963 }
1964
1965 static void
1966 WRITE_REGISTER_USHORT(uint16_t *reg, uint16_t val)
1967 {
1968         bus_space_write_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1969 }
1970
1971 static uint16_t
1972 READ_REGISTER_USHORT(reg)
1973         uint16_t                *reg;
1974 {
1975         return (bus_space_read_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1976 }
1977
1978 static void
1979 WRITE_REGISTER_ULONG(reg, val)
1980         uint32_t                *reg;
1981         uint32_t                val;
1982 {
1983         bus_space_write_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1984 }
1985
1986 static uint32_t
1987 READ_REGISTER_ULONG(reg)
1988         uint32_t                *reg;
1989 {
1990         return (bus_space_read_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1991 }
1992
1993 static uint8_t
1994 READ_REGISTER_UCHAR(uint8_t *reg)
1995 {
1996         return (bus_space_read_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1997 }
1998
1999 static void
2000 WRITE_REGISTER_UCHAR(uint8_t *reg, uint8_t val)
2001 {
2002         bus_space_write_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
2003 }
2004
2005 static int64_t
2006 _allmul(a, b)
2007         int64_t                 a;
2008         int64_t                 b;
2009 {
2010         return (a * b);
2011 }
2012
2013 static int64_t
2014 _alldiv(a, b)
2015         int64_t                 a;
2016         int64_t                 b;
2017 {
2018         return (a / b);
2019 }
2020
2021 static int64_t
2022 _allrem(a, b)
2023         int64_t                 a;
2024         int64_t                 b;
2025 {
2026         return (a % b);
2027 }
2028
2029 static uint64_t
2030 _aullmul(a, b)
2031         uint64_t                a;
2032         uint64_t                b;
2033 {
2034         return (a * b);
2035 }
2036
2037 static uint64_t
2038 _aulldiv(a, b)
2039         uint64_t                a;
2040         uint64_t                b;
2041 {
2042         return (a / b);
2043 }
2044
2045 static uint64_t
2046 _aullrem(a, b)
2047         uint64_t                a;
2048         uint64_t                b;
2049 {
2050         return (a % b);
2051 }
2052
2053 static int64_t
2054 _allshl(int64_t a, uint8_t b)
2055 {
2056         return (a << b);
2057 }
2058
2059 static uint64_t
2060 _aullshl(uint64_t a, uint8_t b)
2061 {
2062         return (a << b);
2063 }
2064
2065 static int64_t
2066 _allshr(int64_t a, uint8_t b)
2067 {
2068         return (a >> b);
2069 }
2070
2071 static uint64_t
2072 _aullshr(uint64_t a, uint8_t b)
2073 {
2074         return (a >> b);
2075 }
2076
2077 static slist_entry *
2078 ntoskrnl_pushsl(head, entry)
2079         slist_header            *head;
2080         slist_entry             *entry;
2081 {
2082         slist_entry             *oldhead;
2083
2084         oldhead = head->slh_list.slh_next;
2085         entry->sl_next = head->slh_list.slh_next;
2086         head->slh_list.slh_next = entry;
2087         head->slh_list.slh_depth++;
2088         head->slh_list.slh_seq++;
2089
2090         return (oldhead);
2091 }
2092
2093 static void
2094 InitializeSListHead(head)
2095         slist_header            *head;
2096 {
2097         memset(head, 0, sizeof(*head));
2098 }
2099
2100 static slist_entry *
2101 ntoskrnl_popsl(head)
2102         slist_header            *head;
2103 {
2104         slist_entry             *first;
2105
2106         first = head->slh_list.slh_next;
2107         if (first != NULL) {
2108                 head->slh_list.slh_next = first->sl_next;
2109                 head->slh_list.slh_depth--;
2110                 head->slh_list.slh_seq++;
2111         }
2112
2113         return (first);
2114 }
2115
2116 /*
2117  * We need this to make lookaside lists work for amd64.
2118  * We pass a pointer to ExAllocatePoolWithTag() the lookaside
2119  * list structure. For amd64 to work right, this has to be a
2120  * pointer to the wrapped version of the routine, not the
2121  * original. Letting the Windows driver invoke the original
2122  * function directly will result in a convention calling
2123  * mismatch and a pretty crash. On x86, this effectively
2124  * becomes a no-op since ipt_func and ipt_wrap are the same.
2125  */
2126
2127 static funcptr
2128 ntoskrnl_findwrap(func)
2129         funcptr                 func;
2130 {
2131         image_patch_table       *patch;
2132
2133         patch = ntoskrnl_functbl;
2134         while (patch->ipt_func != NULL) {
2135                 if ((funcptr)patch->ipt_func == func)
2136                         return ((funcptr)patch->ipt_wrap);
2137                 patch++;
2138         }
2139
2140         return (NULL);
2141 }
2142
2143 static void
2144 ExInitializePagedLookasideList(paged_lookaside_list *lookaside,
2145         lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2146         uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2147 {
2148         bzero((char *)lookaside, sizeof(paged_lookaside_list));
2149
2150         if (size < sizeof(slist_entry))
2151                 lookaside->nll_l.gl_size = sizeof(slist_entry);
2152         else
2153                 lookaside->nll_l.gl_size = size;
2154         lookaside->nll_l.gl_tag = tag;
2155         if (allocfunc == NULL)
2156                 lookaside->nll_l.gl_allocfunc =
2157                     ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2158         else
2159                 lookaside->nll_l.gl_allocfunc = allocfunc;
2160
2161         if (freefunc == NULL)
2162                 lookaside->nll_l.gl_freefunc =
2163                     ntoskrnl_findwrap((funcptr)ExFreePool);
2164         else
2165                 lookaside->nll_l.gl_freefunc = freefunc;
2166
2167 #ifdef __i386__
2168         KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2169 #endif
2170
2171         lookaside->nll_l.gl_type = NonPagedPool;
2172         lookaside->nll_l.gl_depth = depth;
2173         lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2174 }
2175
2176 static void
2177 ExDeletePagedLookasideList(lookaside)
2178         paged_lookaside_list   *lookaside;
2179 {
2180         void                    *buf;
2181         void            (*freefunc)(void *);
2182
2183         freefunc = lookaside->nll_l.gl_freefunc;
2184         while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2185                 MSCALL1(freefunc, buf);
2186 }
2187
2188 static void
2189 ExInitializeNPagedLookasideList(npaged_lookaside_list *lookaside,
2190         lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2191         uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2192 {
2193         bzero((char *)lookaside, sizeof(npaged_lookaside_list));
2194
2195         if (size < sizeof(slist_entry))
2196                 lookaside->nll_l.gl_size = sizeof(slist_entry);
2197         else
2198                 lookaside->nll_l.gl_size = size;
2199         lookaside->nll_l.gl_tag = tag;
2200         if (allocfunc == NULL)
2201                 lookaside->nll_l.gl_allocfunc =
2202                     ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2203         else
2204                 lookaside->nll_l.gl_allocfunc = allocfunc;
2205
2206         if (freefunc == NULL)
2207                 lookaside->nll_l.gl_freefunc =
2208                     ntoskrnl_findwrap((funcptr)ExFreePool);
2209         else
2210                 lookaside->nll_l.gl_freefunc = freefunc;
2211
2212 #ifdef __i386__
2213         KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2214 #endif
2215
2216         lookaside->nll_l.gl_type = NonPagedPool;
2217         lookaside->nll_l.gl_depth = depth;
2218         lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2219 }
2220
2221 static void
2222 ExDeleteNPagedLookasideList(lookaside)
2223         npaged_lookaside_list   *lookaside;
2224 {
2225         void                    *buf;
2226         void            (*freefunc)(void *);
2227
2228         freefunc = lookaside->nll_l.gl_freefunc;
2229         while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2230                 MSCALL1(freefunc, buf);
2231 }
2232
2233 slist_entry *
2234 InterlockedPushEntrySList(head, entry)
2235         slist_header            *head;
2236         slist_entry             *entry;
2237 {
2238         slist_entry             *oldhead;
2239
2240         mtx_lock_spin(&ntoskrnl_interlock);
2241         oldhead = ntoskrnl_pushsl(head, entry);
2242         mtx_unlock_spin(&ntoskrnl_interlock);
2243
2244         return (oldhead);
2245 }
2246
2247 slist_entry *
2248 InterlockedPopEntrySList(head)
2249         slist_header            *head;
2250 {
2251         slist_entry             *first;
2252
2253         mtx_lock_spin(&ntoskrnl_interlock);
2254         first = ntoskrnl_popsl(head);
2255         mtx_unlock_spin(&ntoskrnl_interlock);
2256
2257         return (first);
2258 }
2259
2260 static slist_entry *
2261 ExInterlockedPushEntrySList(head, entry, lock)
2262         slist_header            *head;
2263         slist_entry             *entry;
2264         kspin_lock              *lock;
2265 {
2266         return (InterlockedPushEntrySList(head, entry));
2267 }
2268
2269 static slist_entry *
2270 ExInterlockedPopEntrySList(head, lock)
2271         slist_header            *head;
2272         kspin_lock              *lock;
2273 {
2274         return (InterlockedPopEntrySList(head));
2275 }
2276
2277 uint16_t
2278 ExQueryDepthSList(head)
2279         slist_header            *head;
2280 {
2281         uint16_t                depth;
2282
2283         mtx_lock_spin(&ntoskrnl_interlock);
2284         depth = head->slh_list.slh_depth;
2285         mtx_unlock_spin(&ntoskrnl_interlock);
2286
2287         return (depth);
2288 }
2289
2290 void
2291 KeInitializeSpinLock(lock)
2292         kspin_lock              *lock;
2293 {
2294         *lock = 0;
2295 }
2296
2297 #ifdef __i386__
2298 void
2299 KefAcquireSpinLockAtDpcLevel(lock)
2300         kspin_lock              *lock;
2301 {
2302 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2303         int                     i = 0;
2304 #endif
2305
2306         while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) {
2307                 /* sit and spin */;
2308 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2309                 i++;
2310                 if (i > 200000000)
2311                         panic("DEADLOCK!");
2312 #endif
2313         }
2314 }
2315
2316 void
2317 KefReleaseSpinLockFromDpcLevel(lock)
2318         kspin_lock              *lock;
2319 {
2320         atomic_store_rel_int((volatile u_int *)lock, 0);
2321 }
2322
2323 uint8_t
2324 KeAcquireSpinLockRaiseToDpc(kspin_lock *lock)
2325 {
2326         uint8_t                 oldirql;
2327
2328         if (KeGetCurrentIrql() > DISPATCH_LEVEL)
2329                 panic("IRQL_NOT_LESS_THAN_OR_EQUAL");
2330
2331         KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
2332         KeAcquireSpinLockAtDpcLevel(lock);
2333
2334         return (oldirql);
2335 }
2336 #else
2337 void
2338 KeAcquireSpinLockAtDpcLevel(kspin_lock *lock)
2339 {
2340         while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0)
2341                 /* sit and spin */;
2342 }
2343
2344 void
2345 KeReleaseSpinLockFromDpcLevel(kspin_lock *lock)
2346 {
2347         atomic_store_rel_int((volatile u_int *)lock, 0);
2348 }
2349 #endif /* __i386__ */
2350
2351 uintptr_t
2352 InterlockedExchange(dst, val)
2353         volatile uint32_t       *dst;
2354         uintptr_t               val;
2355 {
2356         uintptr_t               r;
2357
2358         mtx_lock_spin(&ntoskrnl_interlock);
2359         r = *dst;
2360         *dst = val;
2361         mtx_unlock_spin(&ntoskrnl_interlock);
2362
2363         return (r);
2364 }
2365
2366 static uint32_t
2367 InterlockedIncrement(addend)
2368         volatile uint32_t       *addend;
2369 {
2370         atomic_add_long((volatile u_long *)addend, 1);
2371         return (*addend);
2372 }
2373
2374 static uint32_t
2375 InterlockedDecrement(addend)
2376         volatile uint32_t       *addend;
2377 {
2378         atomic_subtract_long((volatile u_long *)addend, 1);
2379         return (*addend);
2380 }
2381
2382 static void
2383 ExInterlockedAddLargeStatistic(addend, inc)
2384         uint64_t                *addend;
2385         uint32_t                inc;
2386 {
2387         mtx_lock_spin(&ntoskrnl_interlock);
2388         *addend += inc;
2389         mtx_unlock_spin(&ntoskrnl_interlock);
2390 };
2391
2392 mdl *
2393 IoAllocateMdl(void *vaddr, uint32_t len, uint8_t secondarybuf,
2394         uint8_t chargequota, irp *iopkt)
2395 {
2396         mdl                     *m;
2397         int                     zone = 0;
2398
2399         if (MmSizeOfMdl(vaddr, len) > MDL_ZONE_SIZE)
2400                 m = ExAllocatePoolWithTag(NonPagedPool,
2401                     MmSizeOfMdl(vaddr, len), 0);
2402         else {
2403                 m = uma_zalloc(mdl_zone, M_NOWAIT | M_ZERO);
2404                 zone++;
2405         }
2406
2407         if (m == NULL)
2408                 return (NULL);
2409
2410         MmInitializeMdl(m, vaddr, len);
2411
2412         /*
2413          * MmInitializMdl() clears the flags field, so we
2414          * have to set this here. If the MDL came from the
2415          * MDL UMA zone, tag it so we can release it to
2416          * the right place later.
2417          */
2418         if (zone)
2419                 m->mdl_flags = MDL_ZONE_ALLOCED;
2420
2421         if (iopkt != NULL) {
2422                 if (secondarybuf == TRUE) {
2423                         mdl                     *last;
2424                         last = iopkt->irp_mdl;
2425                         while (last->mdl_next != NULL)
2426                                 last = last->mdl_next;
2427                         last->mdl_next = m;
2428                 } else {
2429                         if (iopkt->irp_mdl != NULL)
2430                                 panic("leaking an MDL in IoAllocateMdl()");
2431                         iopkt->irp_mdl = m;
2432                 }
2433         }
2434
2435         return (m);
2436 }
2437
2438 void
2439 IoFreeMdl(m)
2440         mdl                     *m;
2441 {
2442         if (m == NULL)
2443                 return;
2444
2445         if (m->mdl_flags & MDL_ZONE_ALLOCED)
2446                 uma_zfree(mdl_zone, m);
2447         else
2448                 ExFreePool(m);
2449 }
2450
2451 static void *
2452 MmAllocateContiguousMemory(size, highest)
2453         uint32_t                size;
2454         uint64_t                highest;
2455 {
2456         void *addr;
2457         size_t pagelength = roundup(size, PAGE_SIZE);
2458
2459         addr = ExAllocatePoolWithTag(NonPagedPool, pagelength, 0);
2460
2461         return (addr);
2462 }
2463
2464 static void *
2465 MmAllocateContiguousMemorySpecifyCache(size, lowest, highest,
2466     boundary, cachetype)
2467         uint32_t                size;
2468         uint64_t                lowest;
2469         uint64_t                highest;
2470         uint64_t                boundary;
2471         enum nt_caching_type    cachetype;
2472 {
2473         vm_memattr_t            memattr;
2474         void                    *ret;
2475
2476         switch (cachetype) {
2477         case MmNonCached:
2478                 memattr = VM_MEMATTR_UNCACHEABLE;
2479                 break;
2480         case MmWriteCombined:
2481                 memattr = VM_MEMATTR_WRITE_COMBINING;
2482                 break;
2483         case MmNonCachedUnordered:
2484                 memattr = VM_MEMATTR_UNCACHEABLE;
2485                 break;
2486         case MmCached:
2487         case MmHardwareCoherentCached:
2488         case MmUSWCCached:
2489         default:
2490                 memattr = VM_MEMATTR_DEFAULT;
2491                 break;
2492         }
2493
2494         ret = (void *)kmem_alloc_contig(kernel_arena, size, M_ZERO | M_NOWAIT,
2495             lowest, highest, PAGE_SIZE, boundary, memattr);
2496         if (ret != NULL)
2497                 malloc_type_allocated(M_DEVBUF, round_page(size));
2498         return (ret);
2499 }
2500
2501 static void
2502 MmFreeContiguousMemory(base)
2503         void                    *base;
2504 {
2505         ExFreePool(base);
2506 }
2507
2508 static void
2509 MmFreeContiguousMemorySpecifyCache(base, size, cachetype)
2510         void                    *base;
2511         uint32_t                size;
2512         enum nt_caching_type    cachetype;
2513 {
2514         contigfree(base, size, M_DEVBUF);
2515 }
2516
2517 static uint32_t
2518 MmSizeOfMdl(vaddr, len)
2519         void                    *vaddr;
2520         size_t                  len;
2521 {
2522         uint32_t                l;
2523
2524         l = sizeof(struct mdl) +
2525             (sizeof(vm_offset_t *) * SPAN_PAGES(vaddr, len));
2526
2527         return (l);
2528 }
2529
2530 /*
2531  * The Microsoft documentation says this routine fills in the
2532  * page array of an MDL with the _physical_ page addresses that
2533  * comprise the buffer, but we don't really want to do that here.
2534  * Instead, we just fill in the page array with the kernel virtual
2535  * addresses of the buffers.
2536  */
2537 void
2538 MmBuildMdlForNonPagedPool(m)
2539         mdl                     *m;
2540 {
2541         vm_offset_t             *mdl_pages;
2542         int                     pagecnt, i;
2543
2544         pagecnt = SPAN_PAGES(m->mdl_byteoffset, m->mdl_bytecount);
2545
2546         if (pagecnt > (m->mdl_size - sizeof(mdl)) / sizeof(vm_offset_t *))
2547                 panic("not enough pages in MDL to describe buffer");
2548
2549         mdl_pages = MmGetMdlPfnArray(m);
2550
2551         for (i = 0; i < pagecnt; i++)
2552                 *mdl_pages = (vm_offset_t)m->mdl_startva + (i * PAGE_SIZE);
2553
2554         m->mdl_flags |= MDL_SOURCE_IS_NONPAGED_POOL;
2555         m->mdl_mappedsystemva = MmGetMdlVirtualAddress(m);
2556 }
2557
2558 static void *
2559 MmMapLockedPages(mdl *buf, uint8_t accessmode)
2560 {
2561         buf->mdl_flags |= MDL_MAPPED_TO_SYSTEM_VA;
2562         return (MmGetMdlVirtualAddress(buf));
2563 }
2564
2565 static void *
2566 MmMapLockedPagesSpecifyCache(mdl *buf, uint8_t accessmode, uint32_t cachetype,
2567         void *vaddr, uint32_t bugcheck, uint32_t prio)
2568 {
2569         return (MmMapLockedPages(buf, accessmode));
2570 }
2571
2572 static void
2573 MmUnmapLockedPages(vaddr, buf)
2574         void                    *vaddr;
2575         mdl                     *buf;
2576 {
2577         buf->mdl_flags &= ~MDL_MAPPED_TO_SYSTEM_VA;
2578 }
2579
2580 /*
2581  * This function has a problem in that it will break if you
2582  * compile this module without PAE and try to use it on a PAE
2583  * kernel. Unfortunately, there's no way around this at the
2584  * moment. It's slightly less broken that using pmap_kextract().
2585  * You'd think the virtual memory subsystem would help us out
2586  * here, but it doesn't.
2587  */
2588
2589 static uint64_t
2590 MmGetPhysicalAddress(void *base)
2591 {
2592         return (pmap_extract(kernel_map->pmap, (vm_offset_t)base));
2593 }
2594
2595 void *
2596 MmGetSystemRoutineAddress(ustr)
2597         unicode_string          *ustr;
2598 {
2599         ansi_string             astr;
2600
2601         if (RtlUnicodeStringToAnsiString(&astr, ustr, TRUE))
2602                 return (NULL);
2603         return (ndis_get_routine_address(ntoskrnl_functbl, astr.as_buf));
2604 }
2605
2606 uint8_t
2607 MmIsAddressValid(vaddr)
2608         void                    *vaddr;
2609 {
2610         if (pmap_extract(kernel_map->pmap, (vm_offset_t)vaddr))
2611                 return (TRUE);
2612
2613         return (FALSE);
2614 }
2615
2616 void *
2617 MmMapIoSpace(paddr, len, cachetype)
2618         uint64_t                paddr;
2619         uint32_t                len;
2620         uint32_t                cachetype;
2621 {
2622         devclass_t              nexus_class;
2623         device_t                *nexus_devs, devp;
2624         int                     nexus_count = 0;
2625         device_t                matching_dev = NULL;
2626         struct resource         *res;
2627         int                     i;
2628         vm_offset_t             v;
2629
2630         /* There will always be at least one nexus. */
2631
2632         nexus_class = devclass_find("nexus");
2633         devclass_get_devices(nexus_class, &nexus_devs, &nexus_count);
2634
2635         for (i = 0; i < nexus_count; i++) {
2636                 devp = nexus_devs[i];
2637                 matching_dev = ntoskrnl_finddev(devp, paddr, &res);
2638                 if (matching_dev)
2639                         break;
2640         }
2641
2642         free(nexus_devs, M_TEMP);
2643
2644         if (matching_dev == NULL)
2645                 return (NULL);
2646
2647         v = (vm_offset_t)rman_get_virtual(res);
2648         if (paddr > rman_get_start(res))
2649                 v += paddr - rman_get_start(res);
2650
2651         return ((void *)v);
2652 }
2653
2654 void
2655 MmUnmapIoSpace(vaddr, len)
2656         void                    *vaddr;
2657         size_t                  len;
2658 {
2659 }
2660
2661
2662 static device_t
2663 ntoskrnl_finddev(dev, paddr, res)
2664         device_t                dev;
2665         uint64_t                paddr;
2666         struct resource         **res;
2667 {
2668         device_t                *children = NULL;
2669         device_t                matching_dev;
2670         int                     childcnt;
2671         struct resource         *r;
2672         struct resource_list    *rl;
2673         struct resource_list_entry      *rle;
2674         uint32_t                flags;
2675         int                     i;
2676
2677         /* We only want devices that have been successfully probed. */
2678
2679         if (device_is_alive(dev) == FALSE)
2680                 return (NULL);
2681
2682         rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
2683         if (rl != NULL) {
2684                 STAILQ_FOREACH(rle, rl, link) {
2685                         r = rle->res;
2686
2687                         if (r == NULL)
2688                                 continue;
2689
2690                         flags = rman_get_flags(r);
2691
2692                         if (rle->type == SYS_RES_MEMORY &&
2693                             paddr >= rman_get_start(r) &&
2694                             paddr <= rman_get_end(r)) {
2695                                 if (!(flags & RF_ACTIVE))
2696                                         bus_activate_resource(dev,
2697                                             SYS_RES_MEMORY, 0, r);
2698                                 *res = r;
2699                                 return (dev);
2700                         }
2701                 }
2702         }
2703
2704         /*
2705          * If this device has children, do another
2706          * level of recursion to inspect them.
2707          */
2708
2709         device_get_children(dev, &children, &childcnt);
2710
2711         for (i = 0; i < childcnt; i++) {
2712                 matching_dev = ntoskrnl_finddev(children[i], paddr, res);
2713                 if (matching_dev != NULL) {
2714                         free(children, M_TEMP);
2715                         return (matching_dev);
2716                 }
2717         }
2718
2719
2720         /* Won't somebody please think of the children! */
2721
2722         if (children != NULL)
2723                 free(children, M_TEMP);
2724
2725         return (NULL);
2726 }
2727
2728 /*
2729  * Workitems are unlike DPCs, in that they run in a user-mode thread
2730  * context rather than at DISPATCH_LEVEL in kernel context. In our
2731  * case we run them in kernel context anyway.
2732  */
2733 static void
2734 ntoskrnl_workitem_thread(arg)
2735         void                    *arg;
2736 {
2737         kdpc_queue              *kq;
2738         list_entry              *l;
2739         io_workitem             *iw;
2740         uint8_t                 irql;
2741
2742         kq = arg;
2743
2744         InitializeListHead(&kq->kq_disp);
2745         kq->kq_td = curthread;
2746         kq->kq_exit = 0;
2747         KeInitializeSpinLock(&kq->kq_lock);
2748         KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
2749
2750         while (1) {
2751                 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
2752
2753                 KeAcquireSpinLock(&kq->kq_lock, &irql);
2754
2755                 if (kq->kq_exit) {
2756                         kq->kq_exit = 0;
2757                         KeReleaseSpinLock(&kq->kq_lock, irql);
2758                         break;
2759                 }
2760
2761                 while (!IsListEmpty(&kq->kq_disp)) {
2762                         l = RemoveHeadList(&kq->kq_disp);
2763                         iw = CONTAINING_RECORD(l,
2764                             io_workitem, iw_listentry);
2765                         InitializeListHead((&iw->iw_listentry));
2766                         if (iw->iw_func == NULL)
2767                                 continue;
2768                         KeReleaseSpinLock(&kq->kq_lock, irql);
2769                         MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx);
2770                         KeAcquireSpinLock(&kq->kq_lock, &irql);
2771                 }
2772
2773                 KeReleaseSpinLock(&kq->kq_lock, irql);
2774         }
2775
2776         kproc_exit(0);
2777         return; /* notreached */
2778 }
2779
2780 static ndis_status
2781 RtlCharToInteger(src, base, val)
2782         const char              *src;
2783         uint32_t                base;
2784         uint32_t                *val;
2785 {
2786         int negative = 0;
2787         uint32_t res;
2788
2789         if (!src || !val)
2790                 return (STATUS_ACCESS_VIOLATION);
2791         while (*src != '\0' && *src <= ' ')
2792                 src++;
2793         if (*src == '+')
2794                 src++;
2795         else if (*src == '-') {
2796                 src++;
2797                 negative = 1;
2798         }
2799         if (base == 0) {
2800                 base = 10;
2801                 if (*src == '0') {
2802                         src++;
2803                         if (*src == 'b') {
2804                                 base = 2;
2805                                 src++;
2806                         } else if (*src == 'o') {
2807                                 base = 8;
2808                                 src++;
2809                         } else if (*src == 'x') {
2810                                 base = 16;
2811                                 src++;
2812                         }
2813                 }
2814         } else if (!(base == 2 || base == 8 || base == 10 || base == 16))
2815                 return (STATUS_INVALID_PARAMETER);
2816
2817         for (res = 0; *src; src++) {
2818                 int v;
2819                 if (isdigit(*src))
2820                         v = *src - '0';
2821                 else if (isxdigit(*src))
2822                         v = tolower(*src) - 'a' + 10;
2823                 else
2824                         v = base;
2825                 if (v >= base)
2826                         return (STATUS_INVALID_PARAMETER);
2827                 res = res * base + v;
2828         }
2829         *val = negative ? -res : res;
2830         return (STATUS_SUCCESS);
2831 }
2832
2833 static void
2834 ntoskrnl_destroy_workitem_threads(void)
2835 {
2836         kdpc_queue              *kq;
2837         int                     i;
2838
2839         for (i = 0; i < WORKITEM_THREADS; i++) {
2840                 kq = wq_queues + i;
2841                 kq->kq_exit = 1;
2842                 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2843                 while (kq->kq_exit)
2844                         tsleep(kq->kq_td->td_proc, PWAIT, "waitiw", hz/10);
2845         }
2846 }
2847
2848 io_workitem *
2849 IoAllocateWorkItem(dobj)
2850         device_object           *dobj;
2851 {
2852         io_workitem             *iw;
2853
2854         iw = uma_zalloc(iw_zone, M_NOWAIT);
2855         if (iw == NULL)
2856                 return (NULL);
2857
2858         InitializeListHead(&iw->iw_listentry);
2859         iw->iw_dobj = dobj;
2860
2861         mtx_lock(&ntoskrnl_dispatchlock);
2862         iw->iw_idx = wq_idx;
2863         WORKIDX_INC(wq_idx);
2864         mtx_unlock(&ntoskrnl_dispatchlock);
2865
2866         return (iw);
2867 }
2868
2869 void
2870 IoFreeWorkItem(iw)
2871         io_workitem             *iw;
2872 {
2873         uma_zfree(iw_zone, iw);
2874 }
2875
2876 void
2877 IoQueueWorkItem(iw, iw_func, qtype, ctx)
2878         io_workitem             *iw;
2879         io_workitem_func        iw_func;
2880         uint32_t                qtype;
2881         void                    *ctx;
2882 {
2883         kdpc_queue              *kq;
2884         list_entry              *l;
2885         io_workitem             *cur;
2886         uint8_t                 irql;
2887
2888         kq = wq_queues + iw->iw_idx;
2889
2890         KeAcquireSpinLock(&kq->kq_lock, &irql);
2891
2892         /*
2893          * Traverse the list and make sure this workitem hasn't
2894          * already been inserted. Queuing the same workitem
2895          * twice will hose the list but good.
2896          */
2897
2898         l = kq->kq_disp.nle_flink;
2899         while (l != &kq->kq_disp) {
2900                 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2901                 if (cur == iw) {
2902                         /* Already queued -- do nothing. */
2903                         KeReleaseSpinLock(&kq->kq_lock, irql);
2904                         return;
2905                 }
2906                 l = l->nle_flink;
2907         }
2908
2909         iw->iw_func = iw_func;
2910         iw->iw_ctx = ctx;
2911
2912         InsertTailList((&kq->kq_disp), (&iw->iw_listentry));
2913         KeReleaseSpinLock(&kq->kq_lock, irql);
2914
2915         KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2916 }
2917
2918 static void
2919 ntoskrnl_workitem(dobj, arg)
2920         device_object           *dobj;
2921         void                    *arg;
2922 {
2923         io_workitem             *iw;
2924         work_queue_item         *w;
2925         work_item_func          f;
2926
2927         iw = arg;
2928         w = (work_queue_item *)dobj;
2929         f = (work_item_func)w->wqi_func;
2930         uma_zfree(iw_zone, iw);
2931         MSCALL2(f, w, w->wqi_ctx);
2932 }
2933
2934 /*
2935  * The ExQueueWorkItem() API is deprecated in Windows XP. Microsoft
2936  * warns that it's unsafe and to use IoQueueWorkItem() instead. The
2937  * problem with ExQueueWorkItem() is that it can't guard against
2938  * the condition where a driver submits a job to the work queue and
2939  * is then unloaded before the job is able to run. IoQueueWorkItem()
2940  * acquires a reference to the device's device_object via the
2941  * object manager and retains it until after the job has completed,
2942  * which prevents the driver from being unloaded before the job
2943  * runs. (We don't currently support this behavior, though hopefully
2944  * that will change once the object manager API is fleshed out a bit.)
2945  *
2946  * Having said all that, the ExQueueWorkItem() API remains, because
2947  * there are still other parts of Windows that use it, including
2948  * NDIS itself: NdisScheduleWorkItem() calls ExQueueWorkItem().
2949  * We fake up the ExQueueWorkItem() API on top of our implementation
2950  * of IoQueueWorkItem(). Workitem thread #3 is reserved exclusively
2951  * for ExQueueWorkItem() jobs, and we pass a pointer to the work
2952  * queue item (provided by the caller) in to IoAllocateWorkItem()
2953  * instead of the device_object. We need to save this pointer so
2954  * we can apply a sanity check: as with the DPC queue and other
2955  * workitem queues, we can't allow the same work queue item to
2956  * be queued twice. If it's already pending, we silently return
2957  */
2958
2959 void
2960 ExQueueWorkItem(w, qtype)
2961         work_queue_item         *w;
2962         uint32_t                qtype;
2963 {
2964         io_workitem             *iw;
2965         io_workitem_func        iwf;
2966         kdpc_queue              *kq;
2967         list_entry              *l;
2968         io_workitem             *cur;
2969         uint8_t                 irql;
2970
2971
2972         /*
2973          * We need to do a special sanity test to make sure
2974          * the ExQueueWorkItem() API isn't used to queue
2975          * the same workitem twice. Rather than checking the
2976          * io_workitem pointer itself, we test the attached
2977          * device object, which is really a pointer to the
2978          * legacy work queue item structure.
2979          */
2980
2981         kq = wq_queues + WORKITEM_LEGACY_THREAD;
2982         KeAcquireSpinLock(&kq->kq_lock, &irql);
2983         l = kq->kq_disp.nle_flink;
2984         while (l != &kq->kq_disp) {
2985                 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2986                 if (cur->iw_dobj == (device_object *)w) {
2987                         /* Already queued -- do nothing. */
2988                         KeReleaseSpinLock(&kq->kq_lock, irql);
2989                         return;
2990                 }
2991                 l = l->nle_flink;
2992         }
2993         KeReleaseSpinLock(&kq->kq_lock, irql);
2994
2995         iw = IoAllocateWorkItem((device_object *)w);
2996         if (iw == NULL)
2997                 return;
2998
2999         iw->iw_idx = WORKITEM_LEGACY_THREAD;
3000         iwf = (io_workitem_func)ntoskrnl_findwrap((funcptr)ntoskrnl_workitem);
3001         IoQueueWorkItem(iw, iwf, qtype, iw);
3002 }
3003
3004 static void
3005 RtlZeroMemory(dst, len)
3006         void                    *dst;
3007         size_t                  len;
3008 {
3009         bzero(dst, len);
3010 }
3011
3012 static void
3013 RtlSecureZeroMemory(dst, len)
3014         void                    *dst;
3015         size_t                  len;
3016 {
3017         memset(dst, 0, len);
3018 }
3019
3020 static void
3021 RtlFillMemory(void *dst, size_t len, uint8_t c)
3022 {
3023         memset(dst, c, len);
3024 }
3025
3026 static void
3027 RtlMoveMemory(dst, src, len)
3028         void                    *dst;
3029         const void              *src;
3030         size_t                  len;
3031 {
3032         memmove(dst, src, len);
3033 }
3034
3035 static void
3036 RtlCopyMemory(dst, src, len)
3037         void                    *dst;
3038         const void              *src;
3039         size_t                  len;
3040 {
3041         bcopy(src, dst, len);
3042 }
3043
3044 static size_t
3045 RtlCompareMemory(s1, s2, len)
3046         const void              *s1;
3047         const void              *s2;
3048         size_t                  len;
3049 {
3050         size_t                  i;
3051         uint8_t                 *m1, *m2;
3052
3053         m1 = __DECONST(char *, s1);
3054         m2 = __DECONST(char *, s2);
3055
3056         for (i = 0; i < len && m1[i] == m2[i]; i++);
3057         return (i);
3058 }
3059
3060 void
3061 RtlInitAnsiString(dst, src)
3062         ansi_string             *dst;
3063         char                    *src;
3064 {
3065         ansi_string             *a;
3066
3067         a = dst;
3068         if (a == NULL)
3069                 return;
3070         if (src == NULL) {
3071                 a->as_len = a->as_maxlen = 0;
3072                 a->as_buf = NULL;
3073         } else {
3074                 a->as_buf = src;
3075                 a->as_len = a->as_maxlen = strlen(src);
3076         }
3077 }
3078
3079 void
3080 RtlInitUnicodeString(dst, src)
3081         unicode_string          *dst;
3082         uint16_t                *src;
3083 {
3084         unicode_string          *u;
3085         int                     i;
3086
3087         u = dst;
3088         if (u == NULL)
3089                 return;
3090         if (src == NULL) {
3091                 u->us_len = u->us_maxlen = 0;
3092                 u->us_buf = NULL;
3093         } else {
3094                 i = 0;
3095                 while(src[i] != 0)
3096                         i++;
3097                 u->us_buf = src;
3098                 u->us_len = u->us_maxlen = i * 2;
3099         }
3100 }
3101
3102 ndis_status
3103 RtlUnicodeStringToInteger(ustr, base, val)
3104         unicode_string          *ustr;
3105         uint32_t                base;
3106         uint32_t                *val;
3107 {
3108         uint16_t                *uchr;
3109         int                     len, neg = 0;
3110         char                    abuf[64];
3111         char                    *astr;
3112
3113         uchr = ustr->us_buf;
3114         len = ustr->us_len;
3115         bzero(abuf, sizeof(abuf));
3116
3117         if ((char)((*uchr) & 0xFF) == '-') {
3118                 neg = 1;
3119                 uchr++;
3120                 len -= 2;
3121         } else if ((char)((*uchr) & 0xFF) == '+') {
3122                 neg = 0;
3123                 uchr++;
3124                 len -= 2;
3125         }
3126
3127         if (base == 0) {
3128                 if ((char)((*uchr) & 0xFF) == 'b') {
3129                         base = 2;
3130                         uchr++;
3131                         len -= 2;
3132                 } else if ((char)((*uchr) & 0xFF) == 'o') {
3133                         base = 8;
3134                         uchr++;
3135                         len -= 2;
3136                 } else if ((char)((*uchr) & 0xFF) == 'x') {
3137                         base = 16;
3138                         uchr++;
3139                         len -= 2;
3140                 } else
3141                         base = 10;
3142         }
3143
3144         astr = abuf;
3145         if (neg) {
3146                 strcpy(astr, "-");
3147                 astr++;
3148         }
3149
3150         ntoskrnl_unicode_to_ascii(uchr, astr, len);
3151         *val = strtoul(abuf, NULL, base);
3152
3153         return (STATUS_SUCCESS);
3154 }
3155
3156 void
3157 RtlFreeUnicodeString(ustr)
3158         unicode_string          *ustr;
3159 {
3160         if (ustr->us_buf == NULL)
3161                 return;
3162         ExFreePool(ustr->us_buf);
3163         ustr->us_buf = NULL;
3164 }
3165
3166 void
3167 RtlFreeAnsiString(astr)
3168         ansi_string             *astr;
3169 {
3170         if (astr->as_buf == NULL)
3171                 return;
3172         ExFreePool(astr->as_buf);
3173         astr->as_buf = NULL;
3174 }
3175
3176 static int
3177 atoi(str)
3178         const char              *str;
3179 {
3180         return (int)strtol(str, (char **)NULL, 10);
3181 }
3182
3183 static long
3184 atol(str)
3185         const char              *str;
3186 {
3187         return strtol(str, (char **)NULL, 10);
3188 }
3189
3190 static int
3191 rand(void)
3192 {
3193
3194         return (random());
3195 }
3196
3197 static void
3198 srand(unsigned int seed)
3199 {
3200
3201         srandom(seed);
3202 }
3203
3204 static uint8_t
3205 IoIsWdmVersionAvailable(uint8_t major, uint8_t minor)
3206 {
3207         if (major == WDM_MAJOR && minor == WDM_MINOR_WINXP)
3208                 return (TRUE);
3209         return (FALSE);
3210 }
3211
3212 static int32_t
3213 IoOpenDeviceRegistryKey(struct device_object *devobj, uint32_t type,
3214     uint32_t mask, void **key)
3215 {
3216         return (NDIS_STATUS_INVALID_DEVICE_REQUEST);
3217 }
3218
3219 static ndis_status
3220 IoGetDeviceObjectPointer(name, reqaccess, fileobj, devobj)
3221         unicode_string          *name;
3222         uint32_t                reqaccess;
3223         void                    *fileobj;
3224         device_object           *devobj;
3225 {
3226         return (STATUS_SUCCESS);
3227 }
3228
3229 static ndis_status
3230 IoGetDeviceProperty(devobj, regprop, buflen, prop, reslen)
3231         device_object           *devobj;
3232         uint32_t                regprop;
3233         uint32_t                buflen;
3234         void                    *prop;
3235         uint32_t                *reslen;
3236 {
3237         driver_object           *drv;
3238         uint16_t                **name;
3239
3240         drv = devobj->do_drvobj;
3241
3242         switch (regprop) {
3243         case DEVPROP_DRIVER_KEYNAME:
3244                 name = prop;
3245                 *name = drv->dro_drivername.us_buf;
3246                 *reslen = drv->dro_drivername.us_len;
3247                 break;
3248         default:
3249                 return (STATUS_INVALID_PARAMETER_2);
3250                 break;
3251         }
3252
3253         return (STATUS_SUCCESS);
3254 }
3255
3256 static void
3257 KeInitializeMutex(kmutex, level)
3258         kmutant                 *kmutex;
3259         uint32_t                level;
3260 {
3261         InitializeListHead((&kmutex->km_header.dh_waitlisthead));
3262         kmutex->km_abandoned = FALSE;
3263         kmutex->km_apcdisable = 1;
3264         kmutex->km_header.dh_sigstate = 1;
3265         kmutex->km_header.dh_type = DISP_TYPE_MUTANT;
3266         kmutex->km_header.dh_size = sizeof(kmutant) / sizeof(uint32_t);
3267         kmutex->km_ownerthread = NULL;
3268 }
3269
3270 static uint32_t
3271 KeReleaseMutex(kmutant *kmutex, uint8_t kwait)
3272 {
3273         uint32_t                prevstate;
3274
3275         mtx_lock(&ntoskrnl_dispatchlock);
3276         prevstate = kmutex->km_header.dh_sigstate;
3277         if (kmutex->km_ownerthread != curthread) {
3278                 mtx_unlock(&ntoskrnl_dispatchlock);
3279                 return (STATUS_MUTANT_NOT_OWNED);
3280         }
3281
3282         kmutex->km_header.dh_sigstate++;
3283         kmutex->km_abandoned = FALSE;
3284
3285         if (kmutex->km_header.dh_sigstate == 1) {
3286                 kmutex->km_ownerthread = NULL;
3287                 ntoskrnl_waittest(&kmutex->km_header, IO_NO_INCREMENT);
3288         }
3289
3290         mtx_unlock(&ntoskrnl_dispatchlock);
3291
3292         return (prevstate);
3293 }
3294
3295 static uint32_t
3296 KeReadStateMutex(kmutex)
3297         kmutant                 *kmutex;
3298 {
3299         return (kmutex->km_header.dh_sigstate);
3300 }
3301
3302 void
3303 KeInitializeEvent(nt_kevent *kevent, uint32_t type, uint8_t state)
3304 {
3305         InitializeListHead((&kevent->k_header.dh_waitlisthead));
3306         kevent->k_header.dh_sigstate = state;
3307         if (type == EVENT_TYPE_NOTIFY)
3308                 kevent->k_header.dh_type = DISP_TYPE_NOTIFICATION_EVENT;
3309         else
3310                 kevent->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_EVENT;
3311         kevent->k_header.dh_size = sizeof(nt_kevent) / sizeof(uint32_t);
3312 }
3313
3314 uint32_t
3315 KeResetEvent(kevent)
3316         nt_kevent               *kevent;
3317 {
3318         uint32_t                prevstate;
3319
3320         mtx_lock(&ntoskrnl_dispatchlock);
3321         prevstate = kevent->k_header.dh_sigstate;
3322         kevent->k_header.dh_sigstate = FALSE;
3323         mtx_unlock(&ntoskrnl_dispatchlock);
3324
3325         return (prevstate);
3326 }
3327
3328 uint32_t
3329 KeSetEvent(nt_kevent *kevent, uint32_t increment, uint8_t kwait)
3330 {
3331         uint32_t                prevstate;
3332         wait_block              *w;
3333         nt_dispatch_header      *dh;
3334         struct thread           *td;
3335         wb_ext                  *we;
3336
3337         mtx_lock(&ntoskrnl_dispatchlock);
3338         prevstate = kevent->k_header.dh_sigstate;
3339         dh = &kevent->k_header;
3340
3341         if (IsListEmpty(&dh->dh_waitlisthead))
3342                 /*
3343                  * If there's nobody in the waitlist, just set
3344                  * the state to signalled.
3345                  */
3346                 dh->dh_sigstate = 1;
3347         else {
3348                 /*
3349                  * Get the first waiter. If this is a synchronization
3350                  * event, just wake up that one thread (don't bother
3351                  * setting the state to signalled since we're supposed
3352                  * to automatically clear synchronization events anyway).
3353                  *
3354                  * If it's a notification event, or the first
3355                  * waiter is doing a WAITTYPE_ALL wait, go through
3356                  * the full wait satisfaction process.
3357                  */
3358                 w = CONTAINING_RECORD(dh->dh_waitlisthead.nle_flink,
3359                     wait_block, wb_waitlist);
3360                 we = w->wb_ext;
3361                 td = we->we_td;
3362                 if (kevent->k_header.dh_type == DISP_TYPE_NOTIFICATION_EVENT ||
3363                     w->wb_waittype == WAITTYPE_ALL) {
3364                         if (prevstate == 0) {
3365                                 dh->dh_sigstate = 1;
3366                                 ntoskrnl_waittest(dh, increment);
3367                         }
3368                 } else {
3369                         w->wb_awakened |= TRUE;
3370                         cv_broadcastpri(&we->we_cv,
3371                             (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
3372                             w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
3373                 }
3374         }
3375
3376         mtx_unlock(&ntoskrnl_dispatchlock);
3377
3378         return (prevstate);
3379 }
3380
3381 void
3382 KeClearEvent(kevent)
3383         nt_kevent               *kevent;
3384 {
3385         kevent->k_header.dh_sigstate = FALSE;
3386 }
3387
3388 uint32_t
3389 KeReadStateEvent(kevent)
3390         nt_kevent               *kevent;
3391 {
3392         return (kevent->k_header.dh_sigstate);
3393 }
3394
3395 /*
3396  * The object manager in Windows is responsible for managing
3397  * references and access to various types of objects, including
3398  * device_objects, events, threads, timers and so on. However,
3399  * there's a difference in the way objects are handled in user
3400  * mode versus kernel mode.
3401  *
3402  * In user mode (i.e. Win32 applications), all objects are
3403  * managed by the object manager. For example, when you create
3404  * a timer or event object, you actually end up with an
3405  * object_header (for the object manager's bookkeeping
3406  * purposes) and an object body (which contains the actual object
3407  * structure, e.g. ktimer, kevent, etc...). This allows Windows
3408  * to manage resource quotas and to enforce access restrictions
3409  * on basically every kind of system object handled by the kernel.
3410  *
3411  * However, in kernel mode, you only end up using the object
3412  * manager some of the time. For example, in a driver, you create
3413  * a timer object by simply allocating the memory for a ktimer
3414  * structure and initializing it with KeInitializeTimer(). Hence,
3415  * the timer has no object_header and no reference counting or
3416  * security/resource checks are done on it. The assumption in
3417  * this case is that if you're running in kernel mode, you know
3418  * what you're doing, and you're already at an elevated privilege
3419  * anyway.
3420  *
3421  * There are some exceptions to this. The two most important ones
3422  * for our purposes are device_objects and threads. We need to use
3423  * the object manager to do reference counting on device_objects,
3424  * and for threads, you can only get a pointer to a thread's
3425  * dispatch header by using ObReferenceObjectByHandle() on the
3426  * handle returned by PsCreateSystemThread().
3427  */
3428
3429 static ndis_status
3430 ObReferenceObjectByHandle(ndis_handle handle, uint32_t reqaccess, void *otype,
3431         uint8_t accessmode, void **object, void **handleinfo)
3432 {
3433         nt_objref               *nr;
3434
3435         nr = malloc(sizeof(nt_objref), M_DEVBUF, M_NOWAIT|M_ZERO);
3436         if (nr == NULL)
3437                 return (STATUS_INSUFFICIENT_RESOURCES);
3438
3439         InitializeListHead((&nr->no_dh.dh_waitlisthead));
3440         nr->no_obj = handle;
3441         nr->no_dh.dh_type = DISP_TYPE_THREAD;
3442         nr->no_dh.dh_sigstate = 0;
3443         nr->no_dh.dh_size = (uint8_t)(sizeof(struct thread) /
3444             sizeof(uint32_t));
3445         TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link);
3446         *object = nr;
3447
3448         return (STATUS_SUCCESS);
3449 }
3450
3451 static void
3452 ObfDereferenceObject(object)
3453         void                    *object;
3454 {
3455         nt_objref               *nr;
3456
3457         nr = object;
3458         TAILQ_REMOVE(&ntoskrnl_reflist, nr, link);
3459         free(nr, M_DEVBUF);
3460 }
3461
3462 static uint32_t
3463 ZwClose(handle)
3464         ndis_handle             handle;
3465 {
3466         return (STATUS_SUCCESS);
3467 }
3468
3469 static uint32_t
3470 WmiQueryTraceInformation(traceclass, traceinfo, infolen, reqlen, buf)
3471         uint32_t                traceclass;
3472         void                    *traceinfo;
3473         uint32_t                infolen;
3474         uint32_t                reqlen;
3475         void                    *buf;
3476 {
3477         return (STATUS_NOT_FOUND);
3478 }
3479
3480 static uint32_t
3481 WmiTraceMessage(uint64_t loghandle, uint32_t messageflags,
3482         void *guid, uint16_t messagenum, ...)
3483 {
3484         return (STATUS_SUCCESS);
3485 }
3486
3487 static uint32_t
3488 IoWMIRegistrationControl(dobj, action)
3489         device_object           *dobj;
3490         uint32_t                action;
3491 {
3492         return (STATUS_SUCCESS);
3493 }
3494
3495 /*
3496  * This is here just in case the thread returns without calling
3497  * PsTerminateSystemThread().
3498  */
3499 static void
3500 ntoskrnl_thrfunc(arg)
3501         void                    *arg;
3502 {
3503         thread_context          *thrctx;
3504         uint32_t (*tfunc)(void *);
3505         void                    *tctx;
3506         uint32_t                rval;
3507
3508         thrctx = arg;
3509         tfunc = thrctx->tc_thrfunc;
3510         tctx = thrctx->tc_thrctx;
3511         free(thrctx, M_TEMP);
3512
3513         rval = MSCALL1(tfunc, tctx);
3514
3515         PsTerminateSystemThread(rval);
3516         return; /* notreached */
3517 }
3518
3519 static ndis_status
3520 PsCreateSystemThread(handle, reqaccess, objattrs, phandle,
3521         clientid, thrfunc, thrctx)
3522         ndis_handle             *handle;
3523         uint32_t                reqaccess;
3524         void                    *objattrs;
3525         ndis_handle             phandle;
3526         void                    *clientid;
3527         void                    *thrfunc;
3528         void                    *thrctx;
3529 {
3530         int                     error;
3531         thread_context          *tc;
3532         struct proc             *p;
3533
3534         tc = malloc(sizeof(thread_context), M_TEMP, M_NOWAIT);
3535         if (tc == NULL)
3536                 return (STATUS_INSUFFICIENT_RESOURCES);
3537
3538         tc->tc_thrctx = thrctx;
3539         tc->tc_thrfunc = thrfunc;
3540
3541         error = kproc_create(ntoskrnl_thrfunc, tc, &p,
3542             RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Kthread %d", ntoskrnl_kth);
3543
3544         if (error) {
3545                 free(tc, M_TEMP);
3546                 return (STATUS_INSUFFICIENT_RESOURCES);
3547         }
3548
3549         *handle = p;
3550         ntoskrnl_kth++;
3551
3552         return (STATUS_SUCCESS);
3553 }
3554
3555 /*
3556  * In Windows, the exit of a thread is an event that you're allowed
3557  * to wait on, assuming you've obtained a reference to the thread using
3558  * ObReferenceObjectByHandle(). Unfortunately, the only way we can
3559  * simulate this behavior is to register each thread we create in a
3560  * reference list, and if someone holds a reference to us, we poke
3561  * them.
3562  */
3563 static ndis_status
3564 PsTerminateSystemThread(status)
3565         ndis_status             status;
3566 {
3567         struct nt_objref        *nr;
3568
3569         mtx_lock(&ntoskrnl_dispatchlock);
3570         TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) {
3571                 if (nr->no_obj != curthread->td_proc)
3572                         continue;
3573                 nr->no_dh.dh_sigstate = 1;
3574                 ntoskrnl_waittest(&nr->no_dh, IO_NO_INCREMENT);
3575                 break;
3576         }
3577         mtx_unlock(&ntoskrnl_dispatchlock);
3578
3579         ntoskrnl_kth--;
3580
3581         kproc_exit(0);
3582         return (0);     /* notreached */
3583 }
3584
3585 static uint32_t
3586 DbgPrint(char *fmt, ...)
3587 {
3588         va_list                 ap;
3589
3590         if (bootverbose) {
3591                 va_start(ap, fmt);
3592                 vprintf(fmt, ap);
3593                 va_end(ap);
3594         }
3595
3596         return (STATUS_SUCCESS);
3597 }
3598
3599 static void
3600 DbgBreakPoint(void)
3601 {
3602
3603         kdb_enter(KDB_WHY_NDIS, "DbgBreakPoint(): breakpoint");
3604 }
3605
3606 static void
3607 KeBugCheckEx(code, param1, param2, param3, param4)
3608     uint32_t                    code;
3609     u_long                      param1;
3610     u_long                      param2;
3611     u_long                      param3;
3612     u_long                      param4;
3613 {
3614         panic("KeBugCheckEx: STOP 0x%X", code);
3615 }
3616
3617 static void
3618 ntoskrnl_timercall(arg)
3619         void                    *arg;
3620 {
3621         ktimer                  *timer;
3622         struct timeval          tv;
3623         kdpc                    *dpc;
3624
3625         mtx_lock(&ntoskrnl_dispatchlock);
3626
3627         timer = arg;
3628
3629 #ifdef NTOSKRNL_DEBUG_TIMERS
3630         ntoskrnl_timer_fires++;
3631 #endif
3632         ntoskrnl_remove_timer(timer);
3633
3634         /*
3635          * This should never happen, but complain
3636          * if it does.
3637          */
3638
3639         if (timer->k_header.dh_inserted == FALSE) {
3640                 mtx_unlock(&ntoskrnl_dispatchlock);
3641                 printf("NTOS: timer %p fired even though "
3642                     "it was canceled\n", timer);
3643                 return;
3644         }
3645
3646         /* Mark the timer as no longer being on the timer queue. */
3647
3648         timer->k_header.dh_inserted = FALSE;
3649
3650         /* Now signal the object and satisfy any waits on it. */
3651
3652         timer->k_header.dh_sigstate = 1;
3653         ntoskrnl_waittest(&timer->k_header, IO_NO_INCREMENT);
3654
3655         /*
3656          * If this is a periodic timer, re-arm it
3657          * so it will fire again. We do this before
3658          * calling any deferred procedure calls because
3659          * it's possible the DPC might cancel the timer,
3660          * in which case it would be wrong for us to
3661          * re-arm it again afterwards.
3662          */
3663
3664         if (timer->k_period) {
3665                 tv.tv_sec = 0;
3666                 tv.tv_usec = timer->k_period * 1000;
3667                 timer->k_header.dh_inserted = TRUE;
3668                 ntoskrnl_insert_timer(timer, tvtohz(&tv));
3669 #ifdef NTOSKRNL_DEBUG_TIMERS
3670                 ntoskrnl_timer_reloads++;
3671 #endif
3672         }
3673
3674         dpc = timer->k_dpc;
3675
3676         mtx_unlock(&ntoskrnl_dispatchlock);
3677
3678         /* If there's a DPC associated with the timer, queue it up. */
3679
3680         if (dpc != NULL)
3681                 KeInsertQueueDpc(dpc, NULL, NULL);
3682 }
3683
3684 #ifdef NTOSKRNL_DEBUG_TIMERS
3685 static int
3686 sysctl_show_timers(SYSCTL_HANDLER_ARGS)
3687 {
3688         int                     ret;
3689
3690         ret = 0;
3691         ntoskrnl_show_timers();
3692         return (sysctl_handle_int(oidp, &ret, 0, req));
3693 }
3694
3695 static void
3696 ntoskrnl_show_timers()
3697 {
3698         int                     i = 0;
3699         list_entry              *l;
3700
3701         mtx_lock_spin(&ntoskrnl_calllock);
3702         l = ntoskrnl_calllist.nle_flink;
3703         while(l != &ntoskrnl_calllist) {
3704                 i++;
3705                 l = l->nle_flink;
3706         }
3707         mtx_unlock_spin(&ntoskrnl_calllock);
3708
3709         printf("\n");
3710         printf("%d timers available (out of %d)\n", i, NTOSKRNL_TIMEOUTS);
3711         printf("timer sets: %qu\n", ntoskrnl_timer_sets);
3712         printf("timer reloads: %qu\n", ntoskrnl_timer_reloads);
3713         printf("timer cancels: %qu\n", ntoskrnl_timer_cancels);
3714         printf("timer fires: %qu\n", ntoskrnl_timer_fires);
3715         printf("\n");
3716 }
3717 #endif
3718
3719 /*
3720  * Must be called with dispatcher lock held.
3721  */
3722
3723 static void
3724 ntoskrnl_insert_timer(timer, ticks)
3725         ktimer                  *timer;
3726         int                     ticks;
3727 {
3728         callout_entry           *e;
3729         list_entry              *l;
3730         struct callout          *c;
3731
3732         /*
3733          * Try and allocate a timer.
3734          */
3735         mtx_lock_spin(&ntoskrnl_calllock);
3736         if (IsListEmpty(&ntoskrnl_calllist)) {
3737                 mtx_unlock_spin(&ntoskrnl_calllock);
3738 #ifdef NTOSKRNL_DEBUG_TIMERS
3739                 ntoskrnl_show_timers();
3740 #endif
3741                 panic("out of timers!");
3742         }
3743         l = RemoveHeadList(&ntoskrnl_calllist);
3744         mtx_unlock_spin(&ntoskrnl_calllock);
3745
3746         e = CONTAINING_RECORD(l, callout_entry, ce_list);
3747         c = &e->ce_callout;
3748
3749         timer->k_callout = c;
3750
3751         callout_init(c, 1);
3752         callout_reset(c, ticks, ntoskrnl_timercall, timer);
3753 }
3754
3755 static void
3756 ntoskrnl_remove_timer(timer)
3757         ktimer                  *timer;
3758 {
3759         callout_entry           *e;
3760
3761         e = (callout_entry *)timer->k_callout;
3762         callout_stop(timer->k_callout);
3763
3764         mtx_lock_spin(&ntoskrnl_calllock);
3765         InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
3766         mtx_unlock_spin(&ntoskrnl_calllock);
3767 }
3768
3769 void
3770 KeInitializeTimer(timer)
3771         ktimer                  *timer;
3772 {
3773         if (timer == NULL)
3774                 return;
3775
3776         KeInitializeTimerEx(timer,  EVENT_TYPE_NOTIFY);
3777 }
3778
3779 void
3780 KeInitializeTimerEx(timer, type)
3781         ktimer                  *timer;
3782         uint32_t                type;
3783 {
3784         if (timer == NULL)
3785                 return;
3786
3787         bzero((char *)timer, sizeof(ktimer));
3788         InitializeListHead((&timer->k_header.dh_waitlisthead));
3789         timer->k_header.dh_sigstate = FALSE;
3790         timer->k_header.dh_inserted = FALSE;
3791         if (type == EVENT_TYPE_NOTIFY)
3792                 timer->k_header.dh_type = DISP_TYPE_NOTIFICATION_TIMER;
3793         else
3794                 timer->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_TIMER;
3795         timer->k_header.dh_size = sizeof(ktimer) / sizeof(uint32_t);
3796 }
3797
3798 /*
3799  * DPC subsystem. A Windows Defered Procedure Call has the following
3800  * properties:
3801  * - It runs at DISPATCH_LEVEL.
3802  * - It can have one of 3 importance values that control when it
3803  *   runs relative to other DPCs in the queue.
3804  * - On SMP systems, it can be set to run on a specific processor.
3805  * In order to satisfy the last property, we create a DPC thread for
3806  * each CPU in the system and bind it to that CPU. Each thread
3807  * maintains three queues with different importance levels, which
3808  * will be processed in order from lowest to highest.
3809  *
3810  * In Windows, interrupt handlers run as DPCs. (Not to be confused
3811  * with ISRs, which run in interrupt context and can preempt DPCs.)
3812  * ISRs are given the highest importance so that they'll take
3813  * precedence over timers and other things.
3814  */
3815
3816 static void
3817 ntoskrnl_dpc_thread(arg)
3818         void                    *arg;
3819 {
3820         kdpc_queue              *kq;
3821         kdpc                    *d;
3822         list_entry              *l;
3823         uint8_t                 irql;
3824
3825         kq = arg;
3826
3827         InitializeListHead(&kq->kq_disp);
3828         kq->kq_td = curthread;
3829         kq->kq_exit = 0;
3830         kq->kq_running = FALSE;
3831         KeInitializeSpinLock(&kq->kq_lock);
3832         KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
3833         KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE);
3834
3835         /*
3836          * Elevate our priority. DPCs are used to run interrupt
3837          * handlers, and they should trigger as soon as possible
3838          * once scheduled by an ISR.
3839          */
3840
3841         thread_lock(curthread);
3842 #ifdef NTOSKRNL_MULTIPLE_DPCS
3843         sched_bind(curthread, kq->kq_cpu);
3844 #endif
3845         sched_prio(curthread, PRI_MIN_KERN);
3846         thread_unlock(curthread);
3847
3848         while (1) {
3849                 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
3850
3851                 KeAcquireSpinLock(&kq->kq_lock, &irql);
3852
3853                 if (kq->kq_exit) {
3854                         kq->kq_exit = 0;
3855                         KeReleaseSpinLock(&kq->kq_lock, irql);
3856                         break;
3857                 }
3858
3859                 kq->kq_running = TRUE;
3860
3861                 while (!IsListEmpty(&kq->kq_disp)) {
3862                         l = RemoveHeadList((&kq->kq_disp));
3863                         d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3864                         InitializeListHead((&d->k_dpclistentry));
3865                         KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
3866                         MSCALL4(d->k_deferedfunc, d, d->k_deferredctx,
3867                             d->k_sysarg1, d->k_sysarg2);
3868                         KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3869                 }
3870
3871                 kq->kq_running = FALSE;
3872
3873                 KeReleaseSpinLock(&kq->kq_lock, irql);
3874
3875                 KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE);
3876         }
3877
3878         kproc_exit(0);
3879         return; /* notreached */
3880 }
3881
3882 static void
3883 ntoskrnl_destroy_dpc_threads(void)
3884 {
3885         kdpc_queue              *kq;
3886         kdpc                    dpc;
3887         int                     i;
3888
3889         kq = kq_queues;
3890 #ifdef NTOSKRNL_MULTIPLE_DPCS
3891         for (i = 0; i < mp_ncpus; i++) {
3892 #else
3893         for (i = 0; i < 1; i++) {
3894 #endif
3895                 kq += i;
3896
3897                 kq->kq_exit = 1;
3898                 KeInitializeDpc(&dpc, NULL, NULL);
3899                 KeSetTargetProcessorDpc(&dpc, i);
3900                 KeInsertQueueDpc(&dpc, NULL, NULL);
3901                 while (kq->kq_exit)
3902                         tsleep(kq->kq_td->td_proc, PWAIT, "dpcw", hz/10);
3903         }
3904 }
3905
3906 static uint8_t
3907 ntoskrnl_insert_dpc(head, dpc)
3908         list_entry              *head;
3909         kdpc                    *dpc;
3910 {
3911         list_entry              *l;
3912         kdpc                    *d;
3913
3914         l = head->nle_flink;
3915         while (l != head) {
3916                 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3917                 if (d == dpc)
3918                         return (FALSE);
3919                 l = l->nle_flink;
3920         }
3921
3922         if (dpc->k_importance == KDPC_IMPORTANCE_LOW)
3923                 InsertTailList((head), (&dpc->k_dpclistentry));
3924         else
3925                 InsertHeadList((head), (&dpc->k_dpclistentry));
3926
3927         return (TRUE);
3928 }
3929
3930 void
3931 KeInitializeDpc(dpc, dpcfunc, dpcctx)
3932         kdpc                    *dpc;
3933         void                    *dpcfunc;
3934         void                    *dpcctx;
3935 {
3936
3937         if (dpc == NULL)
3938                 return;
3939
3940         dpc->k_deferedfunc = dpcfunc;
3941         dpc->k_deferredctx = dpcctx;
3942         dpc->k_num = KDPC_CPU_DEFAULT;
3943         dpc->k_importance = KDPC_IMPORTANCE_MEDIUM;
3944         InitializeListHead((&dpc->k_dpclistentry));
3945 }
3946
3947 uint8_t
3948 KeInsertQueueDpc(dpc, sysarg1, sysarg2)
3949         kdpc                    *dpc;
3950         void                    *sysarg1;
3951         void                    *sysarg2;
3952 {
3953         kdpc_queue              *kq;
3954         uint8_t                 r;
3955         uint8_t                 irql;
3956
3957         if (dpc == NULL)
3958                 return (FALSE);
3959
3960         kq = kq_queues;
3961
3962 #ifdef NTOSKRNL_MULTIPLE_DPCS
3963         KeRaiseIrql(DISPATCH_LEVEL, &irql);
3964
3965         /*
3966          * By default, the DPC is queued to run on the same CPU
3967          * that scheduled it.
3968          */
3969
3970         if (dpc->k_num == KDPC_CPU_DEFAULT)
3971                 kq += curthread->td_oncpu;
3972         else
3973                 kq += dpc->k_num;
3974         KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3975 #else
3976         KeAcquireSpinLock(&kq->kq_lock, &irql);
3977 #endif
3978
3979         r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc);
3980         if (r == TRUE) {
3981                 dpc->k_sysarg1 = sysarg1;
3982                 dpc->k_sysarg2 = sysarg2;
3983         }
3984         KeReleaseSpinLock(&kq->kq_lock, irql);
3985
3986         if (r == FALSE)
3987                 return (r);
3988
3989         KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
3990
3991         return (r);
3992 }
3993
3994 uint8_t
3995 KeRemoveQueueDpc(dpc)
3996         kdpc                    *dpc;
3997 {
3998         kdpc_queue              *kq;
3999         uint8_t                 irql;
4000
4001         if (dpc == NULL)
4002                 return (FALSE);
4003
4004 #ifdef NTOSKRNL_MULTIPLE_DPCS
4005         KeRaiseIrql(DISPATCH_LEVEL, &irql);
4006
4007         kq = kq_queues + dpc->k_num;
4008
4009         KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
4010 #else
4011         kq = kq_queues;
4012         KeAcquireSpinLock(&kq->kq_lock, &irql);
4013 #endif
4014
4015         if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) {
4016                 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
4017                 KeLowerIrql(irql);
4018                 return (FALSE);
4019         }
4020
4021         RemoveEntryList((&dpc->k_dpclistentry));
4022         InitializeListHead((&dpc->k_dpclistentry));
4023
4024         KeReleaseSpinLock(&kq->kq_lock, irql);
4025
4026         return (TRUE);
4027 }
4028
4029 void
4030 KeSetImportanceDpc(dpc, imp)
4031         kdpc                    *dpc;
4032         uint32_t                imp;
4033 {
4034         if (imp != KDPC_IMPORTANCE_LOW &&
4035             imp != KDPC_IMPORTANCE_MEDIUM &&
4036             imp != KDPC_IMPORTANCE_HIGH)
4037                 return;
4038
4039         dpc->k_importance = (uint8_t)imp;
4040 }
4041
4042 void
4043 KeSetTargetProcessorDpc(kdpc *dpc, uint8_t cpu)
4044 {
4045         if (cpu > mp_ncpus)
4046                 return;
4047
4048         dpc->k_num = cpu;
4049 }
4050
4051 void
4052 KeFlushQueuedDpcs(void)
4053 {
4054         kdpc_queue              *kq;
4055         int                     i;
4056
4057         /*
4058          * Poke each DPC queue and wait
4059          * for them to drain.
4060          */
4061
4062 #ifdef NTOSKRNL_MULTIPLE_DPCS
4063         for (i = 0; i < mp_ncpus; i++) {
4064 #else
4065         for (i = 0; i < 1; i++) {
4066 #endif
4067                 kq = kq_queues + i;
4068                 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
4069                 KeWaitForSingleObject(&kq->kq_done, 0, 0, TRUE, NULL);
4070         }
4071 }
4072
4073 uint32_t
4074 KeGetCurrentProcessorNumber(void)
4075 {
4076         return ((uint32_t)curthread->td_oncpu);
4077 }
4078
4079 uint8_t
4080 KeSetTimerEx(timer, duetime, period, dpc)
4081         ktimer                  *timer;
4082         int64_t                 duetime;
4083         uint32_t                period;
4084         kdpc                    *dpc;
4085 {
4086         struct timeval          tv;
4087         uint64_t                curtime;
4088         uint8_t                 pending;
4089
4090         if (timer == NULL)
4091                 return (FALSE);
4092
4093         mtx_lock(&ntoskrnl_dispatchlock);
4094
4095         if (timer->k_header.dh_inserted == TRUE) {
4096                 ntoskrnl_remove_timer(timer);
4097 #ifdef NTOSKRNL_DEBUG_TIMERS
4098                 ntoskrnl_timer_cancels++;
4099 #endif
4100                 timer->k_header.dh_inserted = FALSE;
4101                 pending = TRUE;
4102         } else
4103                 pending = FALSE;
4104
4105         timer->k_duetime = duetime;
4106         timer->k_period = period;
4107         timer->k_header.dh_sigstate = FALSE;
4108         timer->k_dpc = dpc;
4109
4110         if (duetime < 0) {
4111                 tv.tv_sec = - (duetime) / 10000000;
4112                 tv.tv_usec = (- (duetime) / 10) -
4113                     (tv.tv_sec * 1000000);
4114         } else {
4115                 ntoskrnl_time(&curtime);
4116                 if (duetime < curtime)
4117                         tv.tv_sec = tv.tv_usec = 0;
4118                 else {
4119                         tv.tv_sec = ((duetime) - curtime) / 10000000;
4120                         tv.tv_usec = ((duetime) - curtime) / 10 -
4121                             (tv.tv_sec * 1000000);
4122                 }
4123         }
4124
4125         timer->k_header.dh_inserted = TRUE;
4126         ntoskrnl_insert_timer(timer, tvtohz(&tv));
4127 #ifdef NTOSKRNL_DEBUG_TIMERS
4128         ntoskrnl_timer_sets++;
4129 #endif
4130
4131         mtx_unlock(&ntoskrnl_dispatchlock);
4132
4133         return (pending);
4134 }
4135
4136 uint8_t
4137 KeSetTimer(timer, duetime, dpc)
4138         ktimer                  *timer;
4139         int64_t                 duetime;
4140         kdpc                    *dpc;
4141 {
4142         return (KeSetTimerEx(timer, duetime, 0, dpc));
4143 }
4144
4145 /*
4146  * The Windows DDK documentation seems to say that cancelling
4147  * a timer that has a DPC will result in the DPC also being
4148  * cancelled, but this isn't really the case.
4149  */
4150
4151 uint8_t
4152 KeCancelTimer(timer)
4153         ktimer                  *timer;
4154 {
4155         uint8_t                 pending;
4156
4157         if (timer == NULL)
4158                 return (FALSE);
4159
4160         mtx_lock(&ntoskrnl_dispatchlock);
4161
4162         pending = timer->k_header.dh_inserted;
4163
4164         if (timer->k_header.dh_inserted == TRUE) {
4165                 timer->k_header.dh_inserted = FALSE;
4166                 ntoskrnl_remove_timer(timer);
4167 #ifdef NTOSKRNL_DEBUG_TIMERS
4168                 ntoskrnl_timer_cancels++;
4169 #endif
4170         }
4171
4172         mtx_unlock(&ntoskrnl_dispatchlock);
4173
4174         return (pending);
4175 }
4176
4177 uint8_t
4178 KeReadStateTimer(timer)
4179         ktimer                  *timer;
4180 {
4181         return (timer->k_header.dh_sigstate);
4182 }
4183
4184 static int32_t
4185 KeDelayExecutionThread(uint8_t wait_mode, uint8_t alertable, int64_t *interval)
4186 {
4187         ktimer                  timer;
4188
4189         if (wait_mode != 0)
4190                 panic("invalid wait_mode %d", wait_mode);
4191
4192         KeInitializeTimer(&timer);
4193         KeSetTimer(&timer, *interval, NULL);
4194         KeWaitForSingleObject(&timer, 0, 0, alertable, NULL);
4195
4196         return STATUS_SUCCESS;
4197 }
4198
4199 static uint64_t
4200 KeQueryInterruptTime(void)
4201 {
4202         int ticks;
4203         struct timeval tv;
4204
4205         getmicrouptime(&tv);
4206
4207         ticks = tvtohz(&tv);
4208
4209         return ticks * howmany(10000000, hz);
4210 }
4211
4212 static struct thread *
4213 KeGetCurrentThread(void)
4214 {
4215
4216         return curthread;
4217 }
4218
4219 static int32_t
4220 KeSetPriorityThread(td, pri)
4221         struct thread   *td;
4222         int32_t         pri;
4223 {
4224         int32_t old;
4225
4226         if (td == NULL)
4227                 return LOW_REALTIME_PRIORITY;
4228
4229         if (td->td_priority <= PRI_MIN_KERN)
4230                 old = HIGH_PRIORITY;
4231         else if (td->td_priority >= PRI_MAX_KERN)
4232                 old = LOW_PRIORITY;
4233         else
4234                 old = LOW_REALTIME_PRIORITY;
4235
4236         thread_lock(td);
4237         if (pri == HIGH_PRIORITY)
4238                 sched_prio(td, PRI_MIN_KERN);
4239         if (pri == LOW_REALTIME_PRIORITY)
4240                 sched_prio(td, PRI_MIN_KERN + (PRI_MAX_KERN - PRI_MIN_KERN) / 2);
4241         if (pri == LOW_PRIORITY)
4242                 sched_prio(td, PRI_MAX_KERN);
4243         thread_unlock(td);
4244
4245         return old;
4246 }
4247
4248 static void
4249 dummy()
4250 {
4251         printf("ntoskrnl dummy called...\n");
4252 }
4253
4254
4255 image_patch_table ntoskrnl_functbl[] = {
4256         IMPORT_SFUNC(RtlZeroMemory, 2),
4257         IMPORT_SFUNC(RtlSecureZeroMemory, 2),
4258         IMPORT_SFUNC(RtlFillMemory, 3),
4259         IMPORT_SFUNC(RtlMoveMemory, 3),
4260         IMPORT_SFUNC(RtlCharToInteger, 3),
4261         IMPORT_SFUNC(RtlCopyMemory, 3),
4262         IMPORT_SFUNC(RtlCopyString, 2),
4263         IMPORT_SFUNC(RtlCompareMemory, 3),
4264         IMPORT_SFUNC(RtlEqualUnicodeString, 3),
4265         IMPORT_SFUNC(RtlCopyUnicodeString, 2),
4266         IMPORT_SFUNC(RtlUnicodeStringToAnsiString, 3),
4267         IMPORT_SFUNC(RtlAnsiStringToUnicodeString, 3),
4268         IMPORT_SFUNC(RtlInitAnsiString, 2),
4269         IMPORT_SFUNC_MAP(RtlInitString, RtlInitAnsiString, 2),
4270         IMPORT_SFUNC(RtlInitUnicodeString, 2),
4271         IMPORT_SFUNC(RtlFreeAnsiString, 1),
4272         IMPORT_SFUNC(RtlFreeUnicodeString, 1),
4273         IMPORT_SFUNC(RtlUnicodeStringToInteger, 3),
4274         IMPORT_CFUNC(sprintf, 0),
4275         IMPORT_CFUNC(vsprintf, 0),
4276         IMPORT_CFUNC_MAP(_snprintf, snprintf, 0),
4277         IMPORT_CFUNC_MAP(_vsnprintf, vsnprintf, 0),
4278         IMPORT_CFUNC(DbgPrint, 0),
4279         IMPORT_SFUNC(DbgBreakPoint, 0),
4280         IMPORT_SFUNC(KeBugCheckEx, 5),
4281         IMPORT_CFUNC(strncmp, 0),
4282         IMPORT_CFUNC(strcmp, 0),
4283         IMPORT_CFUNC_MAP(stricmp, strcasecmp, 0),
4284         IMPORT_CFUNC(strncpy, 0),
4285         IMPORT_CFUNC(strcpy, 0),
4286         IMPORT_CFUNC(strlen, 0),
4287         IMPORT_CFUNC_MAP(toupper, ntoskrnl_toupper, 0),
4288         IMPORT_CFUNC_MAP(tolower, ntoskrnl_tolower, 0),
4289         IMPORT_CFUNC_MAP(strstr, ntoskrnl_strstr, 0),
4290         IMPORT_CFUNC_MAP(strncat, ntoskrnl_strncat, 0),
4291         IMPORT_CFUNC_MAP(strchr, index, 0),
4292         IMPORT_CFUNC_MAP(strrchr, rindex, 0),
4293         IMPORT_CFUNC(memcpy, 0),
4294         IMPORT_CFUNC_MAP(memmove, ntoskrnl_memmove, 0),
4295         IMPORT_CFUNC_MAP(memset, ntoskrnl_memset, 0),
4296         IMPORT_CFUNC_MAP(memchr, ntoskrnl_memchr, 0),
4297         IMPORT_SFUNC(IoAllocateDriverObjectExtension, 4),
4298         IMPORT_SFUNC(IoGetDriverObjectExtension, 2),
4299         IMPORT_FFUNC(IofCallDriver, 2),
4300         IMPORT_FFUNC(IofCompleteRequest, 2),
4301         IMPORT_SFUNC(IoAcquireCancelSpinLock, 1),
4302         IMPORT_SFUNC(IoReleaseCancelSpinLock, 1),
4303         IMPORT_SFUNC(IoCancelIrp, 1),
4304         IMPORT_SFUNC(IoConnectInterrupt, 11),
4305         IMPORT_SFUNC(IoDisconnectInterrupt, 1),
4306         IMPORT_SFUNC(IoCreateDevice, 7),
4307         IMPORT_SFUNC(IoDeleteDevice, 1),
4308         IMPORT_SFUNC(IoGetAttachedDevice, 1),
4309         IMPORT_SFUNC(IoAttachDeviceToDeviceStack, 2),
4310         IMPORT_SFUNC(IoDetachDevice, 1),
4311         IMPORT_SFUNC(IoBuildSynchronousFsdRequest, 7),
4312         IMPORT_SFUNC(IoBuildAsynchronousFsdRequest, 6),
4313         IMPORT_SFUNC(IoBuildDeviceIoControlRequest, 9),
4314         IMPORT_SFUNC(IoAllocateIrp, 2),
4315         IMPORT_SFUNC(IoReuseIrp, 2),
4316         IMPORT_SFUNC(IoMakeAssociatedIrp, 2),
4317         IMPORT_SFUNC(IoFreeIrp, 1),
4318         IMPORT_SFUNC(IoInitializeIrp, 3),
4319         IMPORT_SFUNC(KeAcquireInterruptSpinLock, 1),
4320         IMPORT_SFUNC(KeReleaseInterruptSpinLock, 2),
4321         IMPORT_SFUNC(KeSynchronizeExecution, 3),
4322         IMPORT_SFUNC(KeWaitForSingleObject, 5),
4323         IMPORT_SFUNC(KeWaitForMultipleObjects, 8),
4324         IMPORT_SFUNC(_allmul, 4),
4325         IMPORT_SFUNC(_alldiv, 4),
4326         IMPORT_SFUNC(_allrem, 4),
4327         IMPORT_RFUNC(_allshr, 0),
4328         IMPORT_RFUNC(_allshl, 0),
4329         IMPORT_SFUNC(_aullmul, 4),
4330         IMPORT_SFUNC(_aulldiv, 4),
4331         IMPORT_SFUNC(_aullrem, 4),
4332         IMPORT_RFUNC(_aullshr, 0),
4333         IMPORT_RFUNC(_aullshl, 0),
4334         IMPORT_CFUNC(atoi, 0),
4335         IMPORT_CFUNC(atol, 0),
4336         IMPORT_CFUNC(rand, 0),
4337         IMPORT_CFUNC(srand, 0),
4338         IMPORT_SFUNC(WRITE_REGISTER_USHORT, 2),
4339         IMPORT_SFUNC(READ_REGISTER_USHORT, 1),
4340         IMPORT_SFUNC(WRITE_REGISTER_ULONG, 2),
4341         IMPORT_SFUNC(READ_REGISTER_ULONG, 1),
4342         IMPORT_SFUNC(READ_REGISTER_UCHAR, 1),
4343         IMPORT_SFUNC(WRITE_REGISTER_UCHAR, 2),
4344         IMPORT_SFUNC(ExInitializePagedLookasideList, 7),
4345         IMPORT_SFUNC(ExDeletePagedLookasideList, 1),
4346         IMPORT_SFUNC(ExInitializeNPagedLookasideList, 7),
4347         IMPORT_SFUNC(ExDeleteNPagedLookasideList, 1),
4348         IMPORT_FFUNC(InterlockedPopEntrySList, 1),
4349         IMPORT_FFUNC(InitializeSListHead, 1),
4350         IMPORT_FFUNC(InterlockedPushEntrySList, 2),
4351         IMPORT_SFUNC(ExQueryDepthSList, 1),
4352         IMPORT_FFUNC_MAP(ExpInterlockedPopEntrySList,
4353                 InterlockedPopEntrySList, 1),
4354         IMPORT_FFUNC_MAP(ExpInterlockedPushEntrySList,
4355                 InterlockedPushEntrySList, 2),
4356         IMPORT_FFUNC(ExInterlockedPopEntrySList, 2),
4357         IMPORT_FFUNC(ExInterlockedPushEntrySList, 3),
4358         IMPORT_SFUNC(ExAllocatePoolWithTag, 3),
4359         IMPORT_SFUNC(ExFreePoolWithTag, 2),
4360         IMPORT_SFUNC(ExFreePool, 1),
4361 #ifdef __i386__
4362         IMPORT_FFUNC(KefAcquireSpinLockAtDpcLevel, 1),
4363         IMPORT_FFUNC(KefReleaseSpinLockFromDpcLevel,1),
4364         IMPORT_FFUNC(KeAcquireSpinLockRaiseToDpc, 1),
4365 #else
4366         /*
4367          * For AMD64, we can get away with just mapping
4368          * KeAcquireSpinLockRaiseToDpc() directly to KfAcquireSpinLock()
4369          * because the calling conventions end up being the same.
4370          * On i386, we have to be careful because KfAcquireSpinLock()
4371          * is _fastcall but KeAcquireSpinLockRaiseToDpc() isn't.
4372          */
4373         IMPORT_SFUNC(KeAcquireSpinLockAtDpcLevel, 1),
4374         IMPORT_SFUNC(KeReleaseSpinLockFromDpcLevel, 1),
4375         IMPORT_SFUNC_MAP(KeAcquireSpinLockRaiseToDpc, KfAcquireSpinLock, 1),
4376 #endif
4377         IMPORT_SFUNC_MAP(KeReleaseSpinLock, KfReleaseSpinLock, 1),
4378         IMPORT_FFUNC(InterlockedIncrement, 1),
4379         IMPORT_FFUNC(InterlockedDecrement, 1),
4380         IMPORT_FFUNC(InterlockedExchange, 2),
4381         IMPORT_FFUNC(ExInterlockedAddLargeStatistic, 2),
4382         IMPORT_SFUNC(IoAllocateMdl, 5),
4383         IMPORT_SFUNC(IoFreeMdl, 1),
4384         IMPORT_SFUNC(MmAllocateContiguousMemory, 2 + 1),
4385         IMPORT_SFUNC(MmAllocateContiguousMemorySpecifyCache, 5 + 3),
4386         IMPORT_SFUNC(MmFreeContiguousMemory, 1),
4387         IMPORT_SFUNC(MmFreeContiguousMemorySpecifyCache, 3),
4388         IMPORT_SFUNC(MmSizeOfMdl, 1),
4389         IMPORT_SFUNC(MmMapLockedPages, 2),
4390         IMPORT_SFUNC(MmMapLockedPagesSpecifyCache, 6),
4391         IMPORT_SFUNC(MmUnmapLockedPages, 2),
4392         IMPORT_SFUNC(MmBuildMdlForNonPagedPool, 1),
4393         IMPORT_SFUNC(MmGetPhysicalAddress, 1),
4394         IMPORT_SFUNC(MmGetSystemRoutineAddress, 1),
4395         IMPORT_SFUNC(MmIsAddressValid, 1),
4396         IMPORT_SFUNC(MmMapIoSpace, 3 + 1),
4397         IMPORT_SFUNC(MmUnmapIoSpace, 2),
4398         IMPORT_SFUNC(KeInitializeSpinLock, 1),
4399         IMPORT_SFUNC(IoIsWdmVersionAvailable, 2),
4400         IMPORT_SFUNC(IoOpenDeviceRegistryKey, 4),
4401         IMPORT_SFUNC(IoGetDeviceObjectPointer, 4),
4402         IMPORT_SFUNC(IoGetDeviceProperty, 5),
4403         IMPORT_SFUNC(IoAllocateWorkItem, 1),
4404         IMPORT_SFUNC(IoFreeWorkItem, 1),
4405         IMPORT_SFUNC(IoQueueWorkItem, 4),
4406         IMPORT_SFUNC(ExQueueWorkItem, 2),
4407         IMPORT_SFUNC(ntoskrnl_workitem, 2),
4408         IMPORT_SFUNC(KeInitializeMutex, 2),
4409         IMPORT_SFUNC(KeReleaseMutex, 2),
4410         IMPORT_SFUNC(KeReadStateMutex, 1),
4411         IMPORT_SFUNC(KeInitializeEvent, 3),
4412         IMPORT_SFUNC(KeSetEvent, 3),
4413         IMPORT_SFUNC(KeResetEvent, 1),
4414         IMPORT_SFUNC(KeClearEvent, 1),
4415         IMPORT_SFUNC(KeReadStateEvent, 1),
4416         IMPORT_SFUNC(KeInitializeTimer, 1),
4417         IMPORT_SFUNC(KeInitializeTimerEx, 2),
4418         IMPORT_SFUNC(KeSetTimer, 3),
4419         IMPORT_SFUNC(KeSetTimerEx, 4),
4420         IMPORT_SFUNC(KeCancelTimer, 1),
4421         IMPORT_SFUNC(KeReadStateTimer, 1),
4422         IMPORT_SFUNC(KeInitializeDpc, 3),
4423         IMPORT_SFUNC(KeInsertQueueDpc, 3),
4424         IMPORT_SFUNC(KeRemoveQueueDpc, 1),
4425         IMPORT_SFUNC(KeSetImportanceDpc, 2),
4426         IMPORT_SFUNC(KeSetTargetProcessorDpc, 2),
4427         IMPORT_SFUNC(KeFlushQueuedDpcs, 0),
4428         IMPORT_SFUNC(KeGetCurrentProcessorNumber, 1),
4429         IMPORT_SFUNC(ObReferenceObjectByHandle, 6),
4430         IMPORT_FFUNC(ObfDereferenceObject, 1),
4431         IMPORT_SFUNC(ZwClose, 1),
4432         IMPORT_SFUNC(PsCreateSystemThread, 7),
4433         IMPORT_SFUNC(PsTerminateSystemThread, 1),
4434         IMPORT_SFUNC(IoWMIRegistrationControl, 2),
4435         IMPORT_SFUNC(WmiQueryTraceInformation, 5),
4436         IMPORT_CFUNC(WmiTraceMessage, 0),
4437         IMPORT_SFUNC(KeQuerySystemTime, 1),
4438         IMPORT_CFUNC(KeTickCount, 0),
4439         IMPORT_SFUNC(KeDelayExecutionThread, 3),
4440         IMPORT_SFUNC(KeQueryInterruptTime, 0),
4441         IMPORT_SFUNC(KeGetCurrentThread, 0),
4442         IMPORT_SFUNC(KeSetPriorityThread, 2),
4443
4444         /*
4445          * This last entry is a catch-all for any function we haven't
4446          * implemented yet. The PE import list patching routine will
4447          * use it for any function that doesn't have an explicit match
4448          * in this table.
4449          */
4450
4451         { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL },
4452
4453         /* End of list. */
4454
4455         { NULL, NULL, NULL }
4456 };