sys/compat/ndis/subr_ntoskrnl.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-4-Clause
   3  *
   4  * Copyright (c) 2003
   5  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. All advertising materials mentioning features or use of this software
  16  *    must display the following acknowledgement:
  17  *      This product includes software developed by Bill Paul.
  18  * 4. Neither the name of the author nor the names of any co-contributors
  19  *    may be used to endorse or promote products derived from this software
  20  *    without specific prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  26  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  32  * THE POSSIBILITY OF SUCH DAMAGE.
  33  */
  34
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include <sys/ctype.h>
  39 #include <sys/unistd.h>
  40 #include <sys/param.h>
  41 #include <sys/types.h>
  42 #include <sys/errno.h>
  43 #include <sys/systm.h>
  44 #include <sys/malloc.h>
  45 #include <sys/lock.h>
  46 #include <sys/mutex.h>
  47
  48 #include <sys/callout.h>
  49 #include <sys/kdb.h>
  50 #include <sys/kernel.h>
  51 #include <sys/proc.h>
  52 #include <sys/condvar.h>
  53 #include <sys/kthread.h>
  54 #include <sys/module.h>
  55 #include <sys/smp.h>
  56 #include <sys/sched.h>
  57 #include <sys/sysctl.h>
  58
  59 #include <machine/atomic.h>
  60 #include <machine/bus.h>
  61 #include <machine/stdarg.h>
  62 #include <machine/resource.h>
  63
  64 #include <sys/bus.h>
  65 #include <sys/rman.h>
  66
  67 #include <vm/vm.h>
  68 #include <vm/vm_param.h>
  69 #include <vm/pmap.h>
  70 #include <vm/uma.h>
  71 #include <vm/vm_kern.h>
  72 #include <vm/vm_map.h>
  73 #include <vm/vm_extern.h>
  74
  75 #include <compat/ndis/pe_var.h>
  76 #include <compat/ndis/cfg_var.h>
  77 #include <compat/ndis/resource_var.h>
  78 #include <compat/ndis/ntoskrnl_var.h>
  79 #include <compat/ndis/hal_var.h>
  80 #include <compat/ndis/ndis_var.h>
  81
  82 #ifdef NTOSKRNL_DEBUG_TIMERS
  83 static int sysctl_show_timers(SYSCTL_HANDLER_ARGS);
  84
  85 SYSCTL_PROC(_debug, OID_AUTO, ntoskrnl_timers,
  86     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0,
  87     sysctl_show_timers, "I",
  88     "Show ntoskrnl timer stats");
  89 #endif
  90
  91 struct kdpc_queue {
  92         list_entry              kq_disp;
  93         struct thread           *kq_td;
  94         int                     kq_cpu;
  95         int                     kq_exit;
  96         int                     kq_running;
  97         kspin_lock              kq_lock;
  98         nt_kevent               kq_proc;
  99         nt_kevent               kq_done;
 100 };
 101
 102 typedef struct kdpc_queue kdpc_queue;
 103
 104 struct wb_ext {
 105         struct cv               we_cv;
 106         struct thread           *we_td;
 107 };
 108
 109 typedef struct wb_ext wb_ext;
 110
 111 #define NTOSKRNL_TIMEOUTS       256
 112 #ifdef NTOSKRNL_DEBUG_TIMERS
 113 static uint64_t ntoskrnl_timer_fires;
 114 static uint64_t ntoskrnl_timer_sets;
 115 static uint64_t ntoskrnl_timer_reloads;
 116 static uint64_t ntoskrnl_timer_cancels;
 117 #endif
 118
 119 struct callout_entry {
 120         struct callout          ce_callout;
 121         list_entry              ce_list;
 122 };
 123
 124 typedef struct callout_entry callout_entry;
 125
 126 static struct list_entry ntoskrnl_calllist;
 127 static struct mtx ntoskrnl_calllock;
 128 struct kuser_shared_data kuser_shared_data;
 129
 130 static struct list_entry ntoskrnl_intlist;
 131 static kspin_lock ntoskrnl_intlock;
 132
 133 static uint8_t RtlEqualUnicodeString(unicode_string *,
 134         unicode_string *, uint8_t);
 135 static void RtlCopyString(ansi_string *, const ansi_string *);
 136 static void RtlCopyUnicodeString(unicode_string *,
 137         unicode_string *);
 138 static irp *IoBuildSynchronousFsdRequest(uint32_t, device_object *,
 139          void *, uint32_t, uint64_t *, nt_kevent *, io_status_block *);
 140 static irp *IoBuildAsynchronousFsdRequest(uint32_t,
 141         device_object *, void *, uint32_t, uint64_t *, io_status_block *);
 142 static irp *IoBuildDeviceIoControlRequest(uint32_t,
 143         device_object *, void *, uint32_t, void *, uint32_t,
 144         uint8_t, nt_kevent *, io_status_block *);
 145 static irp *IoAllocateIrp(uint8_t, uint8_t);
 146 static void IoReuseIrp(irp *, uint32_t);
 147 static void IoFreeIrp(irp *);
 148 static void IoInitializeIrp(irp *, uint16_t, uint8_t);
 149 static irp *IoMakeAssociatedIrp(irp *, uint8_t);
 150 static uint32_t KeWaitForMultipleObjects(uint32_t,
 151         nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t,
 152         int64_t *, wait_block *);
 153 static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t);
 154 static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *);
 155 static void ntoskrnl_satisfy_multiple_waits(wait_block *);
 156 static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *);
 157 static void ntoskrnl_insert_timer(ktimer *, int);
 158 static void ntoskrnl_remove_timer(ktimer *);
 159 #ifdef NTOSKRNL_DEBUG_TIMERS
 160 static void ntoskrnl_show_timers(void);
 161 #endif
 162 static void ntoskrnl_timercall(void *);
 163 static void ntoskrnl_dpc_thread(void *);
 164 static void ntoskrnl_destroy_dpc_threads(void);
 165 static void ntoskrnl_destroy_workitem_threads(void);
 166 static void ntoskrnl_workitem_thread(void *);
 167 static void ntoskrnl_workitem(device_object *, void *);
 168 static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int);
 169 static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int);
 170 static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *);
 171 static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t);
 172 static uint16_t READ_REGISTER_USHORT(uint16_t *);
 173 static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t);
 174 static uint32_t READ_REGISTER_ULONG(uint32_t *);
 175 static void WRITE_REGISTER_UCHAR(uint8_t *, uint8_t);
 176 static uint8_t READ_REGISTER_UCHAR(uint8_t *);
 177 static int64_t _allmul(int64_t, int64_t);
 178 static int64_t _alldiv(int64_t, int64_t);
 179 static int64_t _allrem(int64_t, int64_t);
 180 static int64_t _allshr(int64_t, uint8_t);
 181 static int64_t _allshl(int64_t, uint8_t);
 182 static uint64_t _aullmul(uint64_t, uint64_t);
 183 static uint64_t _aulldiv(uint64_t, uint64_t);
 184 static uint64_t _aullrem(uint64_t, uint64_t);
 185 static uint64_t _aullshr(uint64_t, uint8_t);
 186 static uint64_t _aullshl(uint64_t, uint8_t);
 187 static slist_entry *ntoskrnl_pushsl(slist_header *, slist_entry *);
 188 static void InitializeSListHead(slist_header *);
 189 static slist_entry *ntoskrnl_popsl(slist_header *);
 190 static void ExFreePoolWithTag(void *, uint32_t);
 191 static void ExInitializePagedLookasideList(paged_lookaside_list *,
 192         lookaside_alloc_func *, lookaside_free_func *,
 193         uint32_t, size_t, uint32_t, uint16_t);
 194 static void ExDeletePagedLookasideList(paged_lookaside_list *);
 195 static void ExInitializeNPagedLookasideList(npaged_lookaside_list *,
 196         lookaside_alloc_func *, lookaside_free_func *,
 197         uint32_t, size_t, uint32_t, uint16_t);
 198 static void ExDeleteNPagedLookasideList(npaged_lookaside_list *);
 199 static slist_entry
 200         *ExInterlockedPushEntrySList(slist_header *,
 201         slist_entry *, kspin_lock *);
 202 static slist_entry
 203         *ExInterlockedPopEntrySList(slist_header *, kspin_lock *);
 204 static uint32_t InterlockedIncrement(volatile uint32_t *);
 205 static uint32_t InterlockedDecrement(volatile uint32_t *);
 206 static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t);
 207 static void *MmAllocateContiguousMemory(uint32_t, uint64_t);
 208 static void *MmAllocateContiguousMemorySpecifyCache(uint32_t,
 209         uint64_t, uint64_t, uint64_t, enum nt_caching_type);
 210 static void MmFreeContiguousMemory(void *);
 211 static void MmFreeContiguousMemorySpecifyCache(void *, uint32_t,
 212         enum nt_caching_type);
 213 static uint32_t MmSizeOfMdl(void *, size_t);
 214 static void *MmMapLockedPages(mdl *, uint8_t);
 215 static void *MmMapLockedPagesSpecifyCache(mdl *,
 216         uint8_t, uint32_t, void *, uint32_t, uint32_t);
 217 static void MmUnmapLockedPages(void *, mdl *);
 218 static device_t ntoskrnl_finddev(device_t, uint64_t, struct resource **);
 219 static void RtlZeroMemory(void *, size_t);
 220 static void RtlSecureZeroMemory(void *, size_t);
 221 static void RtlFillMemory(void *, size_t, uint8_t);
 222 static void RtlMoveMemory(void *, const void *, size_t);
 223 static ndis_status RtlCharToInteger(const char *, uint32_t, uint32_t *);
 224 static void RtlCopyMemory(void *, const void *, size_t);
 225 static size_t RtlCompareMemory(const void *, const void *, size_t);
 226 static ndis_status RtlUnicodeStringToInteger(unicode_string *,
 227         uint32_t, uint32_t *);
 228 static int atoi (const char *);
 229 static long atol (const char *);
 230 static int rand(void);
 231 static void srand(unsigned int);
 232 static void KeQuerySystemTime(uint64_t *);
 233 static uint32_t KeTickCount(void);
 234 static uint8_t IoIsWdmVersionAvailable(uint8_t, uint8_t);
 235 static int32_t IoOpenDeviceRegistryKey(struct device_object *, uint32_t,
 236     uint32_t, void **);
 237 static void ntoskrnl_thrfunc(void *);
 238 static ndis_status PsCreateSystemThread(ndis_handle *,
 239         uint32_t, void *, ndis_handle, void *, void *, void *);
 240 static ndis_status PsTerminateSystemThread(ndis_status);
 241 static ndis_status IoGetDeviceObjectPointer(unicode_string *,
 242         uint32_t, void *, device_object *);
 243 static ndis_status IoGetDeviceProperty(device_object *, uint32_t,
 244         uint32_t, void *, uint32_t *);
 245 static void KeInitializeMutex(kmutant *, uint32_t);
 246 static uint32_t KeReleaseMutex(kmutant *, uint8_t);
 247 static uint32_t KeReadStateMutex(kmutant *);
 248 static ndis_status ObReferenceObjectByHandle(ndis_handle,
 249         uint32_t, void *, uint8_t, void **, void **);
 250 static void ObfDereferenceObject(void *);
 251 static uint32_t ZwClose(ndis_handle);
 252 static uint32_t WmiQueryTraceInformation(uint32_t, void *, uint32_t,
 253         uint32_t, void *);
 254 static uint32_t WmiTraceMessage(uint64_t, uint32_t, void *, uint16_t, ...);
 255 static uint32_t IoWMIRegistrationControl(device_object *, uint32_t);
 256 static void *ntoskrnl_memset(void *, int, size_t);
 257 static void *ntoskrnl_memmove(void *, void *, size_t);
 258 static void *ntoskrnl_memchr(void *, unsigned char, size_t);
 259 static char *ntoskrnl_strstr(char *, char *);
 260 static char *ntoskrnl_strncat(char *, char *, size_t);
 261 static int ntoskrnl_toupper(int);
 262 static int ntoskrnl_tolower(int);
 263 static funcptr ntoskrnl_findwrap(funcptr);
 264 static uint32_t DbgPrint(char *, ...);
 265 static void DbgBreakPoint(void);
 266 static void KeBugCheckEx(uint32_t, u_long, u_long, u_long, u_long);
 267 static int32_t KeDelayExecutionThread(uint8_t, uint8_t, int64_t *);
 268 static int32_t KeSetPriorityThread(struct thread *, int32_t);
 269 static void dummy(void);
 270
 271 static struct mtx ntoskrnl_dispatchlock;
 272 static struct mtx ntoskrnl_interlock;
 273 static kspin_lock ntoskrnl_cancellock;
 274 static int ntoskrnl_kth = 0;
 275 static struct nt_objref_head ntoskrnl_reflist;
 276 static uma_zone_t mdl_zone;
 277 static uma_zone_t iw_zone;
 278 static struct kdpc_queue *kq_queues;
 279 static struct kdpc_queue *wq_queues;
 280 static int wq_idx = 0;
 281
 282 int
 283 ntoskrnl_libinit()
 284 {
 285         image_patch_table       *patch;
 286         int                     error;
 287         struct proc             *p;
 288         kdpc_queue              *kq;
 289         callout_entry           *e;
 290         int                     i;
 291
 292         mtx_init(&ntoskrnl_dispatchlock,
 293             "ntoskrnl dispatch lock", MTX_NDIS_LOCK, MTX_DEF|MTX_RECURSE);
 294         mtx_init(&ntoskrnl_interlock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
 295         KeInitializeSpinLock(&ntoskrnl_cancellock);
 296         KeInitializeSpinLock(&ntoskrnl_intlock);
 297         TAILQ_INIT(&ntoskrnl_reflist);
 298
 299         InitializeListHead(&ntoskrnl_calllist);
 300         InitializeListHead(&ntoskrnl_intlist);
 301         mtx_init(&ntoskrnl_calllock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
 302
 303         kq_queues = ExAllocatePoolWithTag(NonPagedPool,
 304 #ifdef NTOSKRNL_MULTIPLE_DPCS
 305             sizeof(kdpc_queue) * mp_ncpus, 0);
 306 #else
 307             sizeof(kdpc_queue), 0);
 308 #endif
 309
 310         if (kq_queues == NULL)
 311                 return (ENOMEM);
 312
 313         wq_queues = ExAllocatePoolWithTag(NonPagedPool,
 314             sizeof(kdpc_queue) * WORKITEM_THREADS, 0);
 315
 316         if (wq_queues == NULL)
 317                 return (ENOMEM);
 318
 319 #ifdef NTOSKRNL_MULTIPLE_DPCS
 320         bzero((char *)kq_queues, sizeof(kdpc_queue) * mp_ncpus);
 321 #else
 322         bzero((char *)kq_queues, sizeof(kdpc_queue));
 323 #endif
 324         bzero((char *)wq_queues, sizeof(kdpc_queue) * WORKITEM_THREADS);
 325
 326         /*
 327          * Launch the DPC threads.
 328          */
 329
 330 #ifdef NTOSKRNL_MULTIPLE_DPCS
 331         for (i = 0; i < mp_ncpus; i++) {
 332 #else
 333         for (i = 0; i < 1; i++) {
 334 #endif
 335                 kq = kq_queues + i;
 336                 kq->kq_cpu = i;
 337                 error = kproc_create(ntoskrnl_dpc_thread, kq, &p,
 338                     RFHIGHPID, NDIS_KSTACK_PAGES, "Windows DPC %d", i);
 339                 if (error)
 340                         panic("failed to launch DPC thread");
 341         }
 342
 343         /*
 344          * Launch the workitem threads.
 345          */
 346
 347         for (i = 0; i < WORKITEM_THREADS; i++) {
 348                 kq = wq_queues + i;
 349                 error = kproc_create(ntoskrnl_workitem_thread, kq, &p,
 350                     RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Workitem %d", i);
 351                 if (error)
 352                         panic("failed to launch workitem thread");
 353         }
 354
 355         patch = ntoskrnl_functbl;
 356         while (patch->ipt_func != NULL) {
 357                 windrv_wrap((funcptr)patch->ipt_func,
 358                     (funcptr *)&patch->ipt_wrap,
 359                     patch->ipt_argcnt, patch->ipt_ftype);
 360                 patch++;
 361         }
 362
 363         for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) {
 364                 e = ExAllocatePoolWithTag(NonPagedPool,
 365                     sizeof(callout_entry), 0);
 366                 if (e == NULL)
 367                         panic("failed to allocate timeouts");
 368                 mtx_lock_spin(&ntoskrnl_calllock);
 369                 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
 370                 mtx_unlock_spin(&ntoskrnl_calllock);
 371         }
 372
 373         /*
 374          * MDLs are supposed to be variable size (they describe
 375          * buffers containing some number of pages, but we don't
 376          * know ahead of time how many pages that will be). But
 377          * always allocating them off the heap is very slow. As
 378          * a compromise, we create an MDL UMA zone big enough to
 379          * handle any buffer requiring up to 16 pages, and we
 380          * use those for any MDLs for buffers of 16 pages or less
 381          * in size. For buffers larger than that (which we assume
 382          * will be few and far between, we allocate the MDLs off
 383          * the heap.
 384          */
 385
 386         mdl_zone = uma_zcreate("Windows MDL", MDL_ZONE_SIZE,
 387             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 388
 389         iw_zone = uma_zcreate("Windows WorkItem", sizeof(io_workitem),
 390             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 391
 392         return (0);
 393 }
 394
 395 int
 396 ntoskrnl_libfini()
 397 {
 398         image_patch_table       *patch;
 399         callout_entry           *e;
 400         list_entry              *l;
 401
 402         patch = ntoskrnl_functbl;
 403         while (patch->ipt_func != NULL) {
 404                 windrv_unwrap(patch->ipt_wrap);
 405                 patch++;
 406         }
 407
 408         /* Stop the workitem queues. */
 409         ntoskrnl_destroy_workitem_threads();
 410         /* Stop the DPC queues. */
 411         ntoskrnl_destroy_dpc_threads();
 412
 413         ExFreePool(kq_queues);
 414         ExFreePool(wq_queues);
 415
 416         uma_zdestroy(mdl_zone);
 417         uma_zdestroy(iw_zone);
 418
 419         mtx_lock_spin(&ntoskrnl_calllock);
 420         while(!IsListEmpty(&ntoskrnl_calllist)) {
 421                 l = RemoveHeadList(&ntoskrnl_calllist);
 422                 e = CONTAINING_RECORD(l, callout_entry, ce_list);
 423                 mtx_unlock_spin(&ntoskrnl_calllock);
 424                 ExFreePool(e);
 425                 mtx_lock_spin(&ntoskrnl_calllock);
 426         }
 427         mtx_unlock_spin(&ntoskrnl_calllock);
 428
 429         mtx_destroy(&ntoskrnl_dispatchlock);
 430         mtx_destroy(&ntoskrnl_interlock);
 431         mtx_destroy(&ntoskrnl_calllock);
 432
 433         return (0);
 434 }
 435
 436 /*
 437  * We need to be able to reference this externally from the wrapper;
 438  * GCC only generates a local implementation of memset.
 439  */
 440 static void *
 441 ntoskrnl_memset(buf, ch, size)
 442         void                    *buf;
 443         int                     ch;
 444         size_t                  size;
 445 {
 446         return (memset(buf, ch, size));
 447 }
 448
 449 static void *
 450 ntoskrnl_memmove(dst, src, size)
 451         void                    *src;
 452         void                    *dst;
 453         size_t                  size;
 454 {
 455         bcopy(src, dst, size);
 456         return (dst);
 457 }
 458
 459 static void *
 460 ntoskrnl_memchr(void *buf, unsigned char ch, size_t len)
 461 {
 462         if (len != 0) {
 463                 unsigned char *p = buf;
 464
 465                 do {
 466                         if (*p++ == ch)
 467                                 return (p - 1);
 468                 } while (--len != 0);
 469         }
 470         return (NULL);
 471 }
 472
 473 static char *
 474 ntoskrnl_strstr(s, find)
 475         char *s, *find;
 476 {
 477         char c, sc;
 478         size_t len;
 479
 480         if ((c = *find++) != 0) {
 481                 len = strlen(find);
 482                 do {
 483                         do {
 484                                 if ((sc = *s++) == 0)
 485                                         return (NULL);
 486                         } while (sc != c);
 487                 } while (strncmp(s, find, len) != 0);
 488                 s--;
 489         }
 490         return ((char *)s);
 491 }
 492
 493 /* Taken from libc */
 494 static char *
 495 ntoskrnl_strncat(dst, src, n)
 496         char            *dst;
 497         char            *src;
 498         size_t          n;
 499 {
 500         if (n != 0) {
 501                 char *d = dst;
 502                 const char *s = src;
 503
 504                 while (*d != 0)
 505                         d++;
 506                 do {
 507                         if ((*d = *s++) == 0)
 508                                 break;
 509                         d++;
 510                 } while (--n != 0);
 511                 *d = 0;
 512         }
 513         return (dst);
 514 }
 515
 516 static int
 517 ntoskrnl_toupper(c)
 518         int                     c;
 519 {
 520         return (toupper(c));
 521 }
 522
 523 static int
 524 ntoskrnl_tolower(c)
 525         int                     c;
 526 {
 527         return (tolower(c));
 528 }
 529
 530 static uint8_t
 531 RtlEqualUnicodeString(unicode_string *str1, unicode_string *str2,
 532         uint8_t caseinsensitive)
 533 {
 534         int                     i;
 535
 536         if (str1->us_len != str2->us_len)
 537                 return (FALSE);
 538
 539         for (i = 0; i < str1->us_len; i++) {
 540                 if (caseinsensitive == TRUE) {
 541                         if (toupper((char)(str1->us_buf[i] & 0xFF)) !=
 542                             toupper((char)(str2->us_buf[i] & 0xFF)))
 543                                 return (FALSE);
 544                 } else {
 545                         if (str1->us_buf[i] != str2->us_buf[i])
 546                                 return (FALSE);
 547                 }
 548         }
 549
 550         return (TRUE);
 551 }
 552
 553 static void
 554 RtlCopyString(dst, src)
 555         ansi_string             *dst;
 556         const ansi_string       *src;
 557 {
 558         if (src != NULL && src->as_buf != NULL && dst->as_buf != NULL) {
 559                 dst->as_len = min(src->as_len, dst->as_maxlen);
 560                 memcpy(dst->as_buf, src->as_buf, dst->as_len);
 561                 if (dst->as_len < dst->as_maxlen)
 562                         dst->as_buf[dst->as_len] = 0;
 563         } else
 564                 dst->as_len = 0;
 565 }
 566
 567 static void
 568 RtlCopyUnicodeString(dest, src)
 569         unicode_string          *dest;
 570         unicode_string          *src;
 571 {
 572
 573         if (dest->us_maxlen >= src->us_len)
 574                 dest->us_len = src->us_len;
 575         else
 576                 dest->us_len = dest->us_maxlen;
 577         memcpy(dest->us_buf, src->us_buf, dest->us_len);
 578 }
 579
 580 static void
 581 ntoskrnl_ascii_to_unicode(ascii, unicode, len)
 582         char                    *ascii;
 583         uint16_t                *unicode;
 584         int                     len;
 585 {
 586         int                     i;
 587         uint16_t                *ustr;
 588
 589         ustr = unicode;
 590         for (i = 0; i < len; i++) {
 591                 *ustr = (uint16_t)ascii[i];
 592                 ustr++;
 593         }
 594 }
 595
 596 static void
 597 ntoskrnl_unicode_to_ascii(unicode, ascii, len)
 598         uint16_t                *unicode;
 599         char                    *ascii;
 600         int                     len;
 601 {
 602         int                     i;
 603         uint8_t                 *astr;
 604
 605         astr = ascii;
 606         for (i = 0; i < len / 2; i++) {
 607                 *astr = (uint8_t)unicode[i];
 608                 astr++;
 609         }
 610 }
 611
 612 uint32_t
 613 RtlUnicodeStringToAnsiString(ansi_string *dest, unicode_string *src, uint8_t allocate)
 614 {
 615         if (dest == NULL || src == NULL)
 616                 return (STATUS_INVALID_PARAMETER);
 617
 618         dest->as_len = src->us_len / 2;
 619         if (dest->as_maxlen < dest->as_len)
 620                 dest->as_len = dest->as_maxlen;
 621
 622         if (allocate == TRUE) {
 623                 dest->as_buf = ExAllocatePoolWithTag(NonPagedPool,
 624                     (src->us_len / 2) + 1, 0);
 625                 if (dest->as_buf == NULL)
 626                         return (STATUS_INSUFFICIENT_RESOURCES);
 627                 dest->as_len = dest->as_maxlen = src->us_len / 2;
 628         } else {
 629                 dest->as_len = src->us_len / 2; /* XXX */
 630                 if (dest->as_maxlen < dest->as_len)
 631                         dest->as_len = dest->as_maxlen;
 632         }
 633
 634         ntoskrnl_unicode_to_ascii(src->us_buf, dest->as_buf,
 635             dest->as_len * 2);
 636
 637         return (STATUS_SUCCESS);
 638 }
 639
 640 uint32_t
 641 RtlAnsiStringToUnicodeString(unicode_string *dest, ansi_string *src,
 642         uint8_t allocate)
 643 {
 644         if (dest == NULL || src == NULL)
 645                 return (STATUS_INVALID_PARAMETER);
 646
 647         if (allocate == TRUE) {
 648                 dest->us_buf = ExAllocatePoolWithTag(NonPagedPool,
 649                     src->as_len * 2, 0);
 650                 if (dest->us_buf == NULL)
 651                         return (STATUS_INSUFFICIENT_RESOURCES);
 652                 dest->us_len = dest->us_maxlen = strlen(src->as_buf) * 2;
 653         } else {
 654                 dest->us_len = src->as_len * 2; /* XXX */
 655                 if (dest->us_maxlen < dest->us_len)
 656                         dest->us_len = dest->us_maxlen;
 657         }
 658
 659         ntoskrnl_ascii_to_unicode(src->as_buf, dest->us_buf,
 660             dest->us_len / 2);
 661
 662         return (STATUS_SUCCESS);
 663 }
 664
 665 void *
 666 ExAllocatePoolWithTag(pooltype, len, tag)
 667         uint32_t                pooltype;
 668         size_t                  len;
 669         uint32_t                tag;
 670 {
 671         void                    *buf;
 672
 673         buf = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO);
 674         if (buf == NULL)
 675                 return (NULL);
 676
 677         return (buf);
 678 }
 679
 680 static void
 681 ExFreePoolWithTag(buf, tag)
 682         void            *buf;
 683         uint32_t        tag;
 684 {
 685         ExFreePool(buf);
 686 }
 687
 688 void
 689 ExFreePool(buf)
 690         void                    *buf;
 691 {
 692         free(buf, M_DEVBUF);
 693 }
 694
 695 uint32_t
 696 IoAllocateDriverObjectExtension(drv, clid, extlen, ext)
 697         driver_object           *drv;
 698         void                    *clid;
 699         uint32_t                extlen;
 700         void                    **ext;
 701 {
 702         custom_extension        *ce;
 703
 704         ce = ExAllocatePoolWithTag(NonPagedPool, sizeof(custom_extension)
 705             + extlen, 0);
 706
 707         if (ce == NULL)
 708                 return (STATUS_INSUFFICIENT_RESOURCES);
 709
 710         ce->ce_clid = clid;
 711         InsertTailList((&drv->dro_driverext->dre_usrext), (&ce->ce_list));
 712
 713         *ext = (void *)(ce + 1);
 714
 715         return (STATUS_SUCCESS);
 716 }
 717
 718 void *
 719 IoGetDriverObjectExtension(drv, clid)
 720         driver_object           *drv;
 721         void                    *clid;
 722 {
 723         list_entry              *e;
 724         custom_extension        *ce;
 725
 726         /*
 727          * Sanity check. Our dummy bus drivers don't have
 728          * any driver extensions.
 729          */
 730
 731         if (drv->dro_driverext == NULL)
 732                 return (NULL);
 733
 734         e = drv->dro_driverext->dre_usrext.nle_flink;
 735         while (e != &drv->dro_driverext->dre_usrext) {
 736                 ce = (custom_extension *)e;
 737                 if (ce->ce_clid == clid)
 738                         return ((void *)(ce + 1));
 739                 e = e->nle_flink;
 740         }
 741
 742         return (NULL);
 743 }
 744
 745
 746 uint32_t
 747 IoCreateDevice(driver_object *drv, uint32_t devextlen, unicode_string *devname,
 748         uint32_t devtype, uint32_t devchars, uint8_t exclusive,
 749         device_object **newdev)
 750 {
 751         device_object           *dev;
 752
 753         dev = ExAllocatePoolWithTag(NonPagedPool, sizeof(device_object), 0);
 754         if (dev == NULL)
 755                 return (STATUS_INSUFFICIENT_RESOURCES);
 756
 757         dev->do_type = devtype;
 758         dev->do_drvobj = drv;
 759         dev->do_currirp = NULL;
 760         dev->do_flags = 0;
 761
 762         if (devextlen) {
 763                 dev->do_devext = ExAllocatePoolWithTag(NonPagedPool,
 764                     devextlen, 0);
 765
 766                 if (dev->do_devext == NULL) {
 767                         ExFreePool(dev);
 768                         return (STATUS_INSUFFICIENT_RESOURCES);
 769                 }
 770
 771                 bzero(dev->do_devext, devextlen);
 772         } else
 773                 dev->do_devext = NULL;
 774
 775         dev->do_size = sizeof(device_object) + devextlen;
 776         dev->do_refcnt = 1;
 777         dev->do_attacheddev = NULL;
 778         dev->do_nextdev = NULL;
 779         dev->do_devtype = devtype;
 780         dev->do_stacksize = 1;
 781         dev->do_alignreq = 1;
 782         dev->do_characteristics = devchars;
 783         dev->do_iotimer = NULL;
 784         KeInitializeEvent(&dev->do_devlock, EVENT_TYPE_SYNC, TRUE);
 785
 786         /*
 787          * Vpd is used for disk/tape devices,
 788          * but we don't support those. (Yet.)
 789          */
 790         dev->do_vpb = NULL;
 791
 792         dev->do_devobj_ext = ExAllocatePoolWithTag(NonPagedPool,
 793             sizeof(devobj_extension), 0);
 794
 795         if (dev->do_devobj_ext == NULL) {
 796                 if (dev->do_devext != NULL)
 797                         ExFreePool(dev->do_devext);
 798                 ExFreePool(dev);
 799                 return (STATUS_INSUFFICIENT_RESOURCES);
 800         }
 801
 802         dev->do_devobj_ext->dve_type = 0;
 803         dev->do_devobj_ext->dve_size = sizeof(devobj_extension);
 804         dev->do_devobj_ext->dve_devobj = dev;
 805
 806         /*
 807          * Attach this device to the driver object's list
 808          * of devices. Note: this is not the same as attaching
 809          * the device to the device stack. The driver's AddDevice
 810          * routine must explicitly call IoAddDeviceToDeviceStack()
 811          * to do that.
 812          */
 813
 814         if (drv->dro_devobj == NULL) {
 815                 drv->dro_devobj = dev;
 816                 dev->do_nextdev = NULL;
 817         } else {
 818                 dev->do_nextdev = drv->dro_devobj;
 819                 drv->dro_devobj = dev;
 820         }
 821
 822         *newdev = dev;
 823
 824         return (STATUS_SUCCESS);
 825 }
 826
 827 void
 828 IoDeleteDevice(dev)
 829         device_object           *dev;
 830 {
 831         device_object           *prev;
 832
 833         if (dev == NULL)
 834                 return;
 835
 836         if (dev->do_devobj_ext != NULL)
 837                 ExFreePool(dev->do_devobj_ext);
 838
 839         if (dev->do_devext != NULL)
 840                 ExFreePool(dev->do_devext);
 841
 842         /* Unlink the device from the driver's device list. */
 843
 844         prev = dev->do_drvobj->dro_devobj;
 845         if (prev == dev)
 846                 dev->do_drvobj->dro_devobj = dev->do_nextdev;
 847         else {
 848                 while (prev->do_nextdev != dev)
 849                         prev = prev->do_nextdev;
 850                 prev->do_nextdev = dev->do_nextdev;
 851         }
 852
 853         ExFreePool(dev);
 854 }
 855
 856 device_object *
 857 IoGetAttachedDevice(dev)
 858         device_object           *dev;
 859 {
 860         device_object           *d;
 861
 862         if (dev == NULL)
 863                 return (NULL);
 864
 865         d = dev;
 866
 867         while (d->do_attacheddev != NULL)
 868                 d = d->do_attacheddev;
 869
 870         return (d);
 871 }
 872
 873 static irp *
 874 IoBuildSynchronousFsdRequest(func, dobj, buf, len, off, event, status)
 875         uint32_t                func;
 876         device_object           *dobj;
 877         void                    *buf;
 878         uint32_t                len;
 879         uint64_t                *off;
 880         nt_kevent               *event;
 881         io_status_block         *status;
 882 {
 883         irp                     *ip;
 884
 885         ip = IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status);
 886         if (ip == NULL)
 887                 return (NULL);
 888         ip->irp_usrevent = event;
 889
 890         return (ip);
 891 }
 892
 893 static irp *
 894 IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status)
 895         uint32_t                func;
 896         device_object           *dobj;
 897         void                    *buf;
 898         uint32_t                len;
 899         uint64_t                *off;
 900         io_status_block         *status;
 901 {
 902         irp                     *ip;
 903         io_stack_location       *sl;
 904
 905         ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
 906         if (ip == NULL)
 907                 return (NULL);
 908
 909         ip->irp_usriostat = status;
 910         ip->irp_tail.irp_overlay.irp_thread = NULL;
 911
 912         sl = IoGetNextIrpStackLocation(ip);
 913         sl->isl_major = func;
 914         sl->isl_minor = 0;
 915         sl->isl_flags = 0;
 916         sl->isl_ctl = 0;
 917         sl->isl_devobj = dobj;
 918         sl->isl_fileobj = NULL;
 919         sl->isl_completionfunc = NULL;
 920
 921         ip->irp_userbuf = buf;
 922
 923         if (dobj->do_flags & DO_BUFFERED_IO) {
 924                 ip->irp_assoc.irp_sysbuf =
 925                     ExAllocatePoolWithTag(NonPagedPool, len, 0);
 926                 if (ip->irp_assoc.irp_sysbuf == NULL) {
 927                         IoFreeIrp(ip);
 928                         return (NULL);
 929                 }
 930                 bcopy(buf, ip->irp_assoc.irp_sysbuf, len);
 931         }
 932
 933         if (dobj->do_flags & DO_DIRECT_IO) {
 934                 ip->irp_mdl = IoAllocateMdl(buf, len, FALSE, FALSE, ip);
 935                 if (ip->irp_mdl == NULL) {
 936                         if (ip->irp_assoc.irp_sysbuf != NULL)
 937                                 ExFreePool(ip->irp_assoc.irp_sysbuf);
 938                         IoFreeIrp(ip);
 939                         return (NULL);
 940                 }
 941                 ip->irp_userbuf = NULL;
 942                 ip->irp_assoc.irp_sysbuf = NULL;
 943         }
 944
 945         if (func == IRP_MJ_READ) {
 946                 sl->isl_parameters.isl_read.isl_len = len;
 947                 if (off != NULL)
 948                         sl->isl_parameters.isl_read.isl_byteoff = *off;
 949                 else
 950                         sl->isl_parameters.isl_read.isl_byteoff = 0;
 951         }
 952
 953         if (func == IRP_MJ_WRITE) {
 954                 sl->isl_parameters.isl_write.isl_len = len;
 955                 if (off != NULL)
 956                         sl->isl_parameters.isl_write.isl_byteoff = *off;
 957                 else
 958                         sl->isl_parameters.isl_write.isl_byteoff = 0;
 959         }
 960
 961         return (ip);
 962 }
 963
 964 static irp *
 965 IoBuildDeviceIoControlRequest(uint32_t iocode, device_object *dobj, void *ibuf,
 966         uint32_t ilen, void *obuf, uint32_t olen, uint8_t isinternal,
 967         nt_kevent *event, io_status_block *status)
 968 {
 969         irp                     *ip;
 970         io_stack_location       *sl;
 971         uint32_t                buflen;
 972
 973         ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
 974         if (ip == NULL)
 975                 return (NULL);
 976         ip->irp_usrevent = event;
 977         ip->irp_usriostat = status;
 978         ip->irp_tail.irp_overlay.irp_thread = NULL;
 979
 980         sl = IoGetNextIrpStackLocation(ip);
 981         sl->isl_major = isinternal == TRUE ?
 982             IRP_MJ_INTERNAL_DEVICE_CONTROL : IRP_MJ_DEVICE_CONTROL;
 983         sl->isl_minor = 0;
 984         sl->isl_flags = 0;
 985         sl->isl_ctl = 0;
 986         sl->isl_devobj = dobj;
 987         sl->isl_fileobj = NULL;
 988         sl->isl_completionfunc = NULL;
 989         sl->isl_parameters.isl_ioctl.isl_iocode = iocode;
 990         sl->isl_parameters.isl_ioctl.isl_ibuflen = ilen;
 991         sl->isl_parameters.isl_ioctl.isl_obuflen = olen;
 992
 993         switch(IO_METHOD(iocode)) {
 994         case METHOD_BUFFERED:
 995                 if (ilen > olen)
 996                         buflen = ilen;
 997                 else
 998                         buflen = olen;
 999                 if (buflen) {
1000                         ip->irp_assoc.irp_sysbuf =
1001                             ExAllocatePoolWithTag(NonPagedPool, buflen, 0);
1002                         if (ip->irp_assoc.irp_sysbuf == NULL) {
1003                                 IoFreeIrp(ip);
1004                                 return (NULL);
1005                         }
1006                 }
1007                 if (ilen && ibuf != NULL) {
1008                         bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1009                         bzero((char *)ip->irp_assoc.irp_sysbuf + ilen,
1010                             buflen - ilen);
1011                 } else
1012                         bzero(ip->irp_assoc.irp_sysbuf, ilen);
1013                 ip->irp_userbuf = obuf;
1014                 break;
1015         case METHOD_IN_DIRECT:
1016         case METHOD_OUT_DIRECT:
1017                 if (ilen && ibuf != NULL) {
1018                         ip->irp_assoc.irp_sysbuf =
1019                             ExAllocatePoolWithTag(NonPagedPool, ilen, 0);
1020                         if (ip->irp_assoc.irp_sysbuf == NULL) {
1021                                 IoFreeIrp(ip);
1022                                 return (NULL);
1023                         }
1024                         bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1025                 }
1026                 if (olen && obuf != NULL) {
1027                         ip->irp_mdl = IoAllocateMdl(obuf, olen,
1028                             FALSE, FALSE, ip);
1029                         /*
1030                          * Normally we would MmProbeAndLockPages()
1031                          * here, but we don't have to in our
1032                          * imlementation.
1033                          */
1034                 }
1035                 break;
1036         case METHOD_NEITHER:
1037                 ip->irp_userbuf = obuf;
1038                 sl->isl_parameters.isl_ioctl.isl_type3ibuf = ibuf;
1039                 break;
1040         default:
1041                 break;
1042         }
1043
1044         /*
1045          * Ideally, we should associate this IRP with the calling
1046          * thread here.
1047          */
1048
1049         return (ip);
1050 }
1051
1052 static irp *
1053 IoAllocateIrp(uint8_t stsize, uint8_t chargequota)
1054 {
1055         irp                     *i;
1056
1057         i = ExAllocatePoolWithTag(NonPagedPool, IoSizeOfIrp(stsize), 0);
1058         if (i == NULL)
1059                 return (NULL);
1060
1061         IoInitializeIrp(i, IoSizeOfIrp(stsize), stsize);
1062
1063         return (i);
1064 }
1065
1066 static irp *
1067 IoMakeAssociatedIrp(irp *ip, uint8_t stsize)
1068 {
1069         irp                     *associrp;
1070
1071         associrp = IoAllocateIrp(stsize, FALSE);
1072         if (associrp == NULL)
1073                 return (NULL);
1074
1075         mtx_lock(&ntoskrnl_dispatchlock);
1076         associrp->irp_flags |= IRP_ASSOCIATED_IRP;
1077         associrp->irp_tail.irp_overlay.irp_thread =
1078             ip->irp_tail.irp_overlay.irp_thread;
1079         associrp->irp_assoc.irp_master = ip;
1080         mtx_unlock(&ntoskrnl_dispatchlock);
1081
1082         return (associrp);
1083 }
1084
1085 static void
1086 IoFreeIrp(ip)
1087         irp                     *ip;
1088 {
1089         ExFreePool(ip);
1090 }
1091
1092 static void
1093 IoInitializeIrp(irp *io, uint16_t psize, uint8_t ssize)
1094 {
1095         bzero((char *)io, IoSizeOfIrp(ssize));
1096         io->irp_size = psize;
1097         io->irp_stackcnt = ssize;
1098         io->irp_currentstackloc = ssize;
1099         InitializeListHead(&io->irp_thlist);
1100         io->irp_tail.irp_overlay.irp_csl =
1101             (io_stack_location *)(io + 1) + ssize;
1102 }
1103
1104 static void
1105 IoReuseIrp(ip, status)
1106         irp                     *ip;
1107         uint32_t                status;
1108 {
1109         uint8_t                 allocflags;
1110
1111         allocflags = ip->irp_allocflags;
1112         IoInitializeIrp(ip, ip->irp_size, ip->irp_stackcnt);
1113         ip->irp_iostat.isb_status = status;
1114         ip->irp_allocflags = allocflags;
1115 }
1116
1117 void
1118 IoAcquireCancelSpinLock(uint8_t *irql)
1119 {
1120         KeAcquireSpinLock(&ntoskrnl_cancellock, irql);
1121 }
1122
1123 void
1124 IoReleaseCancelSpinLock(uint8_t irql)
1125 {
1126         KeReleaseSpinLock(&ntoskrnl_cancellock, irql);
1127 }
1128
1129 uint8_t
1130 IoCancelIrp(irp *ip)
1131 {
1132         cancel_func             cfunc;
1133         uint8_t                 cancelirql;
1134
1135         IoAcquireCancelSpinLock(&cancelirql);
1136         cfunc = IoSetCancelRoutine(ip, NULL);
1137         ip->irp_cancel = TRUE;
1138         if (cfunc == NULL) {
1139                 IoReleaseCancelSpinLock(cancelirql);
1140                 return (FALSE);
1141         }
1142         ip->irp_cancelirql = cancelirql;
1143         MSCALL2(cfunc, IoGetCurrentIrpStackLocation(ip)->isl_devobj, ip);
1144         return (uint8_t)IoSetCancelValue(ip, TRUE);
1145 }
1146
1147 uint32_t
1148 IofCallDriver(dobj, ip)
1149         device_object           *dobj;
1150         irp                     *ip;
1151 {
1152         driver_object           *drvobj;
1153         io_stack_location       *sl;
1154         uint32_t                status;
1155         driver_dispatch         disp;
1156
1157         drvobj = dobj->do_drvobj;
1158
1159         if (ip->irp_currentstackloc <= 0)
1160                 panic("IoCallDriver(): out of stack locations");
1161
1162         IoSetNextIrpStackLocation(ip);
1163         sl = IoGetCurrentIrpStackLocation(ip);
1164
1165         sl->isl_devobj = dobj;
1166
1167         disp = drvobj->dro_dispatch[sl->isl_major];
1168         status = MSCALL2(disp, dobj, ip);
1169
1170         return (status);
1171 }
1172
1173 void
1174 IofCompleteRequest(irp *ip, uint8_t prioboost)
1175 {
1176         uint32_t                status;
1177         device_object           *dobj;
1178         io_stack_location       *sl;
1179         completion_func         cf;
1180
1181         KASSERT(ip->irp_iostat.isb_status != STATUS_PENDING,
1182             ("incorrect IRP(%p) status (STATUS_PENDING)", ip));
1183
1184         sl = IoGetCurrentIrpStackLocation(ip);
1185         IoSkipCurrentIrpStackLocation(ip);
1186
1187         do {
1188                 if (sl->isl_ctl & SL_PENDING_RETURNED)
1189                         ip->irp_pendingreturned = TRUE;
1190
1191                 if (ip->irp_currentstackloc != (ip->irp_stackcnt + 1))
1192                         dobj = IoGetCurrentIrpStackLocation(ip)->isl_devobj;
1193                 else
1194                         dobj = NULL;
1195
1196                 if (sl->isl_completionfunc != NULL &&
1197                     ((ip->irp_iostat.isb_status == STATUS_SUCCESS &&
1198                     sl->isl_ctl & SL_INVOKE_ON_SUCCESS) ||
1199                     (ip->irp_iostat.isb_status != STATUS_SUCCESS &&
1200                     sl->isl_ctl & SL_INVOKE_ON_ERROR) ||
1201                     (ip->irp_cancel == TRUE &&
1202                     sl->isl_ctl & SL_INVOKE_ON_CANCEL))) {
1203                         cf = sl->isl_completionfunc;
1204                         status = MSCALL3(cf, dobj, ip, sl->isl_completionctx);
1205                         if (status == STATUS_MORE_PROCESSING_REQUIRED)
1206                                 return;
1207                 } else {
1208                         if ((ip->irp_currentstackloc <= ip->irp_stackcnt) &&
1209                             (ip->irp_pendingreturned == TRUE))
1210                                 IoMarkIrpPending(ip);
1211                 }
1212
1213                 /* move to the next.  */
1214                 IoSkipCurrentIrpStackLocation(ip);
1215                 sl++;
1216         } while (ip->irp_currentstackloc <= (ip->irp_stackcnt + 1));
1217
1218         if (ip->irp_usriostat != NULL)
1219                 *ip->irp_usriostat = ip->irp_iostat;
1220         if (ip->irp_usrevent != NULL)
1221                 KeSetEvent(ip->irp_usrevent, prioboost, FALSE);
1222
1223         /* Handle any associated IRPs. */
1224
1225         if (ip->irp_flags & IRP_ASSOCIATED_IRP) {
1226                 uint32_t                masterirpcnt;
1227                 irp                     *masterirp;
1228                 mdl                     *m;
1229
1230                 masterirp = ip->irp_assoc.irp_master;
1231                 masterirpcnt =
1232                     InterlockedDecrement(&masterirp->irp_assoc.irp_irpcnt);
1233
1234                 while ((m = ip->irp_mdl) != NULL) {
1235                         ip->irp_mdl = m->mdl_next;
1236                         IoFreeMdl(m);
1237                 }
1238                 IoFreeIrp(ip);
1239                 if (masterirpcnt == 0)
1240                         IoCompleteRequest(masterirp, IO_NO_INCREMENT);
1241                 return;
1242         }
1243
1244         /* With any luck, these conditions will never arise. */
1245
1246         if (ip->irp_flags & IRP_PAGING_IO) {
1247                 if (ip->irp_mdl != NULL)
1248                         IoFreeMdl(ip->irp_mdl);
1249                 IoFreeIrp(ip);
1250         }
1251 }
1252
1253 void
1254 ntoskrnl_intr(arg)
1255         void                    *arg;
1256 {
1257         kinterrupt              *iobj;
1258         uint8_t                 irql;
1259         uint8_t                 claimed;
1260         list_entry              *l;
1261
1262         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1263         l = ntoskrnl_intlist.nle_flink;
1264         while (l != &ntoskrnl_intlist) {
1265                 iobj = CONTAINING_RECORD(l, kinterrupt, ki_list);
1266                 claimed = MSCALL2(iobj->ki_svcfunc, iobj, iobj->ki_svcctx);
1267                 if (claimed == TRUE)
1268                         break;
1269                 l = l->nle_flink;
1270         }
1271         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1272 }
1273
1274 uint8_t
1275 KeAcquireInterruptSpinLock(iobj)
1276         kinterrupt              *iobj;
1277 {
1278         uint8_t                 irql;
1279         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1280         return (irql);
1281 }
1282
1283 void
1284 KeReleaseInterruptSpinLock(kinterrupt *iobj, uint8_t irql)
1285 {
1286         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1287 }
1288
1289 uint8_t
1290 KeSynchronizeExecution(iobj, syncfunc, syncctx)
1291         kinterrupt              *iobj;
1292         void                    *syncfunc;
1293         void                    *syncctx;
1294 {
1295         uint8_t                 irql;
1296
1297         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1298         MSCALL1(syncfunc, syncctx);
1299         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1300
1301         return (TRUE);
1302 }
1303
1304 /*
1305  * IoConnectInterrupt() is passed only the interrupt vector and
1306  * irql that a device wants to use, but no device-specific tag
1307  * of any kind. This conflicts rather badly with FreeBSD's
1308  * bus_setup_intr(), which needs the device_t for the device
1309  * requesting interrupt delivery. In order to bypass this
1310  * inconsistency, we implement a second level of interrupt
1311  * dispatching on top of bus_setup_intr(). All devices use
1312  * ntoskrnl_intr() as their ISR, and any device requesting
1313  * interrupts will be registered with ntoskrnl_intr()'s interrupt
1314  * dispatch list. When an interrupt arrives, we walk the list
1315  * and invoke all the registered ISRs. This effectively makes all
1316  * interrupts shared, but it's the only way to duplicate the
1317  * semantics of IoConnectInterrupt() and IoDisconnectInterrupt() properly.
1318  */
1319
1320 uint32_t
1321 IoConnectInterrupt(kinterrupt **iobj, void *svcfunc, void *svcctx,
1322         kspin_lock *lock, uint32_t vector, uint8_t irql, uint8_t syncirql,
1323         uint8_t imode, uint8_t shared, uint32_t affinity, uint8_t savefloat)
1324 {
1325         uint8_t                 curirql;
1326
1327         *iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0);
1328         if (*iobj == NULL)
1329                 return (STATUS_INSUFFICIENT_RESOURCES);
1330
1331         (*iobj)->ki_svcfunc = svcfunc;
1332         (*iobj)->ki_svcctx = svcctx;
1333
1334         if (lock == NULL) {
1335                 KeInitializeSpinLock(&(*iobj)->ki_lock_priv);
1336                 (*iobj)->ki_lock = &(*iobj)->ki_lock_priv;
1337         } else
1338                 (*iobj)->ki_lock = lock;
1339
1340         KeAcquireSpinLock(&ntoskrnl_intlock, &curirql);
1341         InsertHeadList((&ntoskrnl_intlist), (&(*iobj)->ki_list));
1342         KeReleaseSpinLock(&ntoskrnl_intlock, curirql);
1343
1344         return (STATUS_SUCCESS);
1345 }
1346
1347 void
1348 IoDisconnectInterrupt(iobj)
1349         kinterrupt              *iobj;
1350 {
1351         uint8_t                 irql;
1352
1353         if (iobj == NULL)
1354                 return;
1355
1356         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1357         RemoveEntryList((&iobj->ki_list));
1358         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1359
1360         ExFreePool(iobj);
1361 }
1362
1363 device_object *
1364 IoAttachDeviceToDeviceStack(src, dst)
1365         device_object           *src;
1366         device_object           *dst;
1367 {
1368         device_object           *attached;
1369
1370         mtx_lock(&ntoskrnl_dispatchlock);
1371         attached = IoGetAttachedDevice(dst);
1372         attached->do_attacheddev = src;
1373         src->do_attacheddev = NULL;
1374         src->do_stacksize = attached->do_stacksize + 1;
1375         mtx_unlock(&ntoskrnl_dispatchlock);
1376
1377         return (attached);
1378 }
1379
1380 void
1381 IoDetachDevice(topdev)
1382         device_object           *topdev;
1383 {
1384         device_object           *tail;
1385
1386         mtx_lock(&ntoskrnl_dispatchlock);
1387
1388         /* First, break the chain. */
1389         tail = topdev->do_attacheddev;
1390         if (tail == NULL) {
1391                 mtx_unlock(&ntoskrnl_dispatchlock);
1392                 return;
1393         }
1394         topdev->do_attacheddev = tail->do_attacheddev;
1395         topdev->do_refcnt--;
1396
1397         /* Now reduce the stacksize count for the takm_il objects. */
1398
1399         tail = topdev->do_attacheddev;
1400         while (tail != NULL) {
1401                 tail->do_stacksize--;
1402                 tail = tail->do_attacheddev;
1403         }
1404
1405         mtx_unlock(&ntoskrnl_dispatchlock);
1406 }
1407
1408 /*
1409  * For the most part, an object is considered signalled if
1410  * dh_sigstate == TRUE. The exception is for mutant objects
1411  * (mutexes), where the logic works like this:
1412  *
1413  * - If the thread already owns the object and sigstate is
1414  *   less than or equal to 0, then the object is considered
1415  *   signalled (recursive acquisition).
1416  * - If dh_sigstate == 1, the object is also considered
1417  *   signalled.
1418  */
1419
1420 static int
1421 ntoskrnl_is_signalled(obj, td)
1422         nt_dispatch_header      *obj;
1423         struct thread           *td;
1424 {
1425         kmutant                 *km;
1426
1427         if (obj->dh_type == DISP_TYPE_MUTANT) {
1428                 km = (kmutant *)obj;
1429                 if ((obj->dh_sigstate <= 0 && km->km_ownerthread == td) ||
1430                     obj->dh_sigstate == 1)
1431                         return (TRUE);
1432                 return (FALSE);
1433         }
1434
1435         if (obj->dh_sigstate > 0)
1436                 return (TRUE);
1437         return (FALSE);
1438 }
1439
1440 static void
1441 ntoskrnl_satisfy_wait(obj, td)
1442         nt_dispatch_header      *obj;
1443         struct thread           *td;
1444 {
1445         kmutant                 *km;
1446
1447         switch (obj->dh_type) {
1448         case DISP_TYPE_MUTANT:
1449                 km = (struct kmutant *)obj;
1450                 obj->dh_sigstate--;
1451                 /*
1452                  * If sigstate reaches 0, the mutex is now
1453                  * non-signalled (the new thread owns it).
1454                  */
1455                 if (obj->dh_sigstate == 0) {
1456                         km->km_ownerthread = td;
1457                         if (km->km_abandoned == TRUE)
1458                                 km->km_abandoned = FALSE;
1459                 }
1460                 break;
1461         /* Synchronization objects get reset to unsignalled. */
1462         case DISP_TYPE_SYNCHRONIZATION_EVENT:
1463         case DISP_TYPE_SYNCHRONIZATION_TIMER:
1464                 obj->dh_sigstate = 0;
1465                 break;
1466         case DISP_TYPE_SEMAPHORE:
1467                 obj->dh_sigstate--;
1468                 break;
1469         default:
1470                 break;
1471         }
1472 }
1473
1474 static void
1475 ntoskrnl_satisfy_multiple_waits(wb)
1476         wait_block              *wb;
1477 {
1478         wait_block              *cur;
1479         struct thread           *td;
1480
1481         cur = wb;
1482         td = wb->wb_kthread;
1483
1484         do {
1485                 ntoskrnl_satisfy_wait(wb->wb_object, td);
1486                 cur->wb_awakened = TRUE;
1487                 cur = cur->wb_next;
1488         } while (cur != wb);
1489 }
1490
1491 /* Always called with dispatcher lock held. */
1492 static void
1493 ntoskrnl_waittest(obj, increment)
1494         nt_dispatch_header      *obj;
1495         uint32_t                increment;
1496 {
1497         wait_block              *w, *next;
1498         list_entry              *e;
1499         struct thread           *td;
1500         wb_ext                  *we;
1501         int                     satisfied;
1502
1503         /*
1504          * Once an object has been signalled, we walk its list of
1505          * wait blocks. If a wait block can be awakened, then satisfy
1506          * waits as necessary and wake the thread.
1507          *
1508          * The rules work like this:
1509          *
1510          * If a wait block is marked as WAITTYPE_ANY, then
1511          * we can satisfy the wait conditions on the current
1512          * object and wake the thread right away. Satisfying
1513          * the wait also has the effect of breaking us out
1514          * of the search loop.
1515          *
1516          * If the object is marked as WAITTYLE_ALL, then the
1517          * wait block will be part of a circularly linked
1518          * list of wait blocks belonging to a waiting thread
1519          * that's sleeping in KeWaitForMultipleObjects(). In
1520          * order to wake the thread, all the objects in the
1521          * wait list must be in the signalled state. If they
1522          * are, we then satisfy all of them and wake the
1523          * thread.
1524          *
1525          */
1526
1527         e = obj->dh_waitlisthead.nle_flink;
1528
1529         while (e != &obj->dh_waitlisthead && obj->dh_sigstate > 0) {
1530                 w = CONTAINING_RECORD(e, wait_block, wb_waitlist);
1531                 we = w->wb_ext;
1532                 td = we->we_td;
1533                 satisfied = FALSE;
1534                 if (w->wb_waittype == WAITTYPE_ANY) {
1535                         /*
1536                          * Thread can be awakened if
1537                          * any wait is satisfied.
1538                          */
1539                         ntoskrnl_satisfy_wait(obj, td);
1540                         satisfied = TRUE;
1541                         w->wb_awakened = TRUE;
1542                 } else {
1543                         /*
1544                          * Thread can only be woken up
1545                          * if all waits are satisfied.
1546                          * If the thread is waiting on multiple
1547                          * objects, they should all be linked
1548                          * through the wb_next pointers in the
1549                          * wait blocks.
1550                          */
1551                         satisfied = TRUE;
1552                         next = w->wb_next;
1553                         while (next != w) {
1554                                 if (ntoskrnl_is_signalled(obj, td) == FALSE) {
1555                                         satisfied = FALSE;
1556                                         break;
1557                                 }
1558                                 next = next->wb_next;
1559                         }
1560                         ntoskrnl_satisfy_multiple_waits(w);
1561                 }
1562
1563                 if (satisfied == TRUE)
1564                         cv_broadcastpri(&we->we_cv,
1565                             (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
1566                             w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
1567
1568                 e = e->nle_flink;
1569         }
1570 }
1571
1572 /*
1573  * Return the number of 100 nanosecond intervals since
1574  * January 1, 1601. (?!?!)
1575  */
1576 void
1577 ntoskrnl_time(tval)
1578         uint64_t                *tval;
1579 {
1580         struct timespec         ts;
1581
1582         nanotime(&ts);
1583         *tval = (uint64_t)ts.tv_nsec / 100 + (uint64_t)ts.tv_sec * 10000000 +
1584             11644473600 * 10000000; /* 100ns ticks from 1601 to 1970 */
1585 }
1586
1587 static void
1588 KeQuerySystemTime(current_time)
1589         uint64_t                *current_time;
1590 {
1591         ntoskrnl_time(current_time);
1592 }
1593
1594 static uint32_t
1595 KeTickCount(void)
1596 {
1597         struct timeval tv;
1598         getmicrouptime(&tv);
1599         return tvtohz(&tv);
1600 }
1601
1602
1603 /*
1604  * KeWaitForSingleObject() is a tricky beast, because it can be used
1605  * with several different object types: semaphores, timers, events,
1606  * mutexes and threads. Semaphores don't appear very often, but the
1607  * other object types are quite common. KeWaitForSingleObject() is
1608  * what's normally used to acquire a mutex, and it can be used to
1609  * wait for a thread termination.
1610  *
1611  * The Windows NDIS API is implemented in terms of Windows kernel
1612  * primitives, and some of the object manipulation is duplicated in
1613  * NDIS. For example, NDIS has timers and events, which are actually
1614  * Windows kevents and ktimers. Now, you're supposed to only use the
1615  * NDIS variants of these objects within the confines of the NDIS API,
1616  * but there are some naughty developers out there who will use
1617  * KeWaitForSingleObject() on NDIS timer and event objects, so we
1618  * have to support that as well. Conseqently, our NDIS timer and event
1619  * code has to be closely tied into our ntoskrnl timer and event code,
1620  * just as it is in Windows.
1621  *
1622  * KeWaitForSingleObject() may do different things for different kinds
1623  * of objects:
1624  *
1625  * - For events, we check if the event has been signalled. If the
1626  *   event is already in the signalled state, we just return immediately,
1627  *   otherwise we wait for it to be set to the signalled state by someone
1628  *   else calling KeSetEvent(). Events can be either synchronization or
1629  *   notification events.
1630  *
1631  * - For timers, if the timer has already fired and the timer is in
1632  *   the signalled state, we just return, otherwise we wait on the
1633  *   timer. Unlike an event, timers get signalled automatically when
1634  *   they expire rather than someone having to trip them manually.
1635  *   Timers initialized with KeInitializeTimer() are always notification
1636  *   events: KeInitializeTimerEx() lets you initialize a timer as
1637  *   either a notification or synchronization event.
1638  *
1639  * - For mutexes, we try to acquire the mutex and if we can't, we wait
1640  *   on the mutex until it's available and then grab it. When a mutex is
1641  *   released, it enters the signalled state, which wakes up one of the
1642  *   threads waiting to acquire it. Mutexes are always synchronization
1643  *   events.
1644  *
1645  * - For threads, the only thing we do is wait until the thread object
1646  *   enters a signalled state, which occurs when the thread terminates.
1647  *   Threads are always notification events.
1648  *
1649  * A notification event wakes up all threads waiting on an object. A
1650  * synchronization event wakes up just one. Also, a synchronization event
1651  * is auto-clearing, which means we automatically set the event back to
1652  * the non-signalled state once the wakeup is done.
1653  */
1654
1655 uint32_t
1656 KeWaitForSingleObject(void *arg, uint32_t reason, uint32_t mode,
1657     uint8_t alertable, int64_t *duetime)
1658 {
1659         wait_block              w;
1660         struct thread           *td = curthread;
1661         struct timeval          tv;
1662         int                     error = 0;
1663         uint64_t                curtime;
1664         wb_ext                  we;
1665         nt_dispatch_header      *obj;
1666
1667         obj = arg;
1668
1669         if (obj == NULL)
1670                 return (STATUS_INVALID_PARAMETER);
1671
1672         mtx_lock(&ntoskrnl_dispatchlock);
1673
1674         cv_init(&we.we_cv, "KeWFS");
1675         we.we_td = td;
1676
1677         /*
1678          * Check to see if this object is already signalled,
1679          * and just return without waiting if it is.
1680          */
1681         if (ntoskrnl_is_signalled(obj, td) == TRUE) {
1682                 /* Sanity check the signal state value. */
1683                 if (obj->dh_sigstate != INT32_MIN) {
1684                         ntoskrnl_satisfy_wait(obj, curthread);
1685                         mtx_unlock(&ntoskrnl_dispatchlock);
1686                         return (STATUS_SUCCESS);
1687                 } else {
1688                         /*
1689                          * There's a limit to how many times we can
1690                          * recursively acquire a mutant. If we hit
1691                          * the limit, something is very wrong.
1692                          */
1693                         if (obj->dh_type == DISP_TYPE_MUTANT) {
1694                                 mtx_unlock(&ntoskrnl_dispatchlock);
1695                                 panic("mutant limit exceeded");
1696                         }
1697                 }
1698         }
1699
1700         bzero((char *)&w, sizeof(wait_block));
1701         w.wb_object = obj;
1702         w.wb_ext = &we;
1703         w.wb_waittype = WAITTYPE_ANY;
1704         w.wb_next = &w;
1705         w.wb_waitkey = 0;
1706         w.wb_awakened = FALSE;
1707         w.wb_oldpri = td->td_priority;
1708
1709         InsertTailList((&obj->dh_waitlisthead), (&w.wb_waitlist));
1710
1711         /*
1712          * The timeout value is specified in 100 nanosecond units
1713          * and can be a positive or negative number. If it's positive,
1714          * then the duetime is absolute, and we need to convert it
1715          * to an absolute offset relative to now in order to use it.
1716          * If it's negative, then the duetime is relative and we
1717          * just have to convert the units.
1718          */
1719
1720         if (duetime != NULL) {
1721                 if (*duetime < 0) {
1722                         tv.tv_sec = - (*duetime) / 10000000;
1723                         tv.tv_usec = (- (*duetime) / 10) -
1724                             (tv.tv_sec * 1000000);
1725                 } else {
1726                         ntoskrnl_time(&curtime);
1727                         if (*duetime < curtime)
1728                                 tv.tv_sec = tv.tv_usec = 0;
1729                         else {
1730                                 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1731                                 tv.tv_usec = ((*duetime) - curtime) / 10 -
1732                                     (tv.tv_sec * 1000000);
1733                         }
1734                 }
1735         }
1736
1737         if (duetime == NULL)
1738                 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1739         else
1740                 error = cv_timedwait(&we.we_cv,
1741                     &ntoskrnl_dispatchlock, tvtohz(&tv));
1742
1743         RemoveEntryList(&w.wb_waitlist);
1744
1745         cv_destroy(&we.we_cv);
1746
1747         /* We timed out. Leave the object alone and return status. */
1748
1749         if (error == EWOULDBLOCK) {
1750                 mtx_unlock(&ntoskrnl_dispatchlock);
1751                 return (STATUS_TIMEOUT);
1752         }
1753
1754         mtx_unlock(&ntoskrnl_dispatchlock);
1755
1756         return (STATUS_SUCCESS);
1757 /*
1758         return (KeWaitForMultipleObjects(1, &obj, WAITTYPE_ALL, reason,
1759             mode, alertable, duetime, &w));
1760 */
1761 }
1762
1763 static uint32_t
1764 KeWaitForMultipleObjects(uint32_t cnt, nt_dispatch_header *obj[], uint32_t wtype,
1765         uint32_t reason, uint32_t mode, uint8_t alertable, int64_t *duetime,
1766         wait_block *wb_array)
1767 {
1768         struct thread           *td = curthread;
1769         wait_block              *whead, *w;
1770         wait_block              _wb_array[MAX_WAIT_OBJECTS];
1771         nt_dispatch_header      *cur;
1772         struct timeval          tv;
1773         int                     i, wcnt = 0, error = 0;
1774         uint64_t                curtime;
1775         struct timespec         t1, t2;
1776         uint32_t                status = STATUS_SUCCESS;
1777         wb_ext                  we;
1778
1779         if (cnt > MAX_WAIT_OBJECTS)
1780                 return (STATUS_INVALID_PARAMETER);
1781         if (cnt > THREAD_WAIT_OBJECTS && wb_array == NULL)
1782                 return (STATUS_INVALID_PARAMETER);
1783
1784         mtx_lock(&ntoskrnl_dispatchlock);
1785
1786         cv_init(&we.we_cv, "KeWFM");
1787         we.we_td = td;
1788
1789         if (wb_array == NULL)
1790                 whead = _wb_array;
1791         else
1792                 whead = wb_array;
1793
1794         bzero((char *)whead, sizeof(wait_block) * cnt);
1795
1796         /* First pass: see if we can satisfy any waits immediately. */
1797
1798         wcnt = 0;
1799         w = whead;
1800
1801         for (i = 0; i < cnt; i++) {
1802                 InsertTailList((&obj[i]->dh_waitlisthead),
1803                     (&w->wb_waitlist));
1804                 w->wb_ext = &we;
1805                 w->wb_object = obj[i];
1806                 w->wb_waittype = wtype;
1807                 w->wb_waitkey = i;
1808                 w->wb_awakened = FALSE;
1809                 w->wb_oldpri = td->td_priority;
1810                 w->wb_next = w + 1;
1811                 w++;
1812                 wcnt++;
1813                 if (ntoskrnl_is_signalled(obj[i], td)) {
1814                         /*
1815                          * There's a limit to how many times
1816                          * we can recursively acquire a mutant.
1817                          * If we hit the limit, something
1818                          * is very wrong.
1819                          */
1820                         if (obj[i]->dh_sigstate == INT32_MIN &&
1821                             obj[i]->dh_type == DISP_TYPE_MUTANT) {
1822                                 mtx_unlock(&ntoskrnl_dispatchlock);
1823                                 panic("mutant limit exceeded");
1824                         }
1825
1826                         /*
1827                          * If this is a WAITTYPE_ANY wait, then
1828                          * satisfy the waited object and exit
1829                          * right now.
1830                          */
1831
1832                         if (wtype == WAITTYPE_ANY) {
1833                                 ntoskrnl_satisfy_wait(obj[i], td);
1834                                 status = STATUS_WAIT_0 + i;
1835                                 goto wait_done;
1836                         } else {
1837                                 w--;
1838                                 wcnt--;
1839                                 w->wb_object = NULL;
1840                                 RemoveEntryList(&w->wb_waitlist);
1841                         }
1842                 }
1843         }
1844
1845         /*
1846          * If this is a WAITTYPE_ALL wait and all objects are
1847          * already signalled, satisfy the waits and exit now.
1848          */
1849
1850         if (wtype == WAITTYPE_ALL && wcnt == 0) {
1851                 for (i = 0; i < cnt; i++)
1852                         ntoskrnl_satisfy_wait(obj[i], td);
1853                 status = STATUS_SUCCESS;
1854                 goto wait_done;
1855         }
1856
1857         /*
1858          * Create a circular waitblock list. The waitcount
1859          * must always be non-zero when we get here.
1860          */
1861
1862         (w - 1)->wb_next = whead;
1863
1864         /* Wait on any objects that aren't yet signalled. */
1865
1866         /* Calculate timeout, if any. */
1867
1868         if (duetime != NULL) {
1869                 if (*duetime < 0) {
1870                         tv.tv_sec = - (*duetime) / 10000000;
1871                         tv.tv_usec = (- (*duetime) / 10) -
1872                             (tv.tv_sec * 1000000);
1873                 } else {
1874                         ntoskrnl_time(&curtime);
1875                         if (*duetime < curtime)
1876                                 tv.tv_sec = tv.tv_usec = 0;
1877                         else {
1878                                 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1879                                 tv.tv_usec = ((*duetime) - curtime) / 10 -
1880                                     (tv.tv_sec * 1000000);
1881                         }
1882                 }
1883         }
1884
1885         while (wcnt) {
1886                 nanotime(&t1);
1887
1888                 if (duetime == NULL)
1889                         cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1890                 else
1891                         error = cv_timedwait(&we.we_cv,
1892                             &ntoskrnl_dispatchlock, tvtohz(&tv));
1893
1894                 /* Wait with timeout expired. */
1895
1896                 if (error) {
1897                         status = STATUS_TIMEOUT;
1898                         goto wait_done;
1899                 }
1900
1901                 nanotime(&t2);
1902
1903                 /* See what's been signalled. */
1904
1905                 w = whead;
1906                 do {
1907                         cur = w->wb_object;
1908                         if (ntoskrnl_is_signalled(cur, td) == TRUE ||
1909                             w->wb_awakened == TRUE) {
1910                                 /* Sanity check the signal state value. */
1911                                 if (cur->dh_sigstate == INT32_MIN &&
1912                                     cur->dh_type == DISP_TYPE_MUTANT) {
1913                                         mtx_unlock(&ntoskrnl_dispatchlock);
1914                                         panic("mutant limit exceeded");
1915                                 }
1916                                 wcnt--;
1917                                 if (wtype == WAITTYPE_ANY) {
1918                                         status = w->wb_waitkey &
1919                                             STATUS_WAIT_0;
1920                                         goto wait_done;
1921                                 }
1922                         }
1923                         w = w->wb_next;
1924                 } while (w != whead);
1925
1926                 /*
1927                  * If all objects have been signalled, or if this
1928                  * is a WAITTYPE_ANY wait and we were woke up by
1929                  * someone, we can bail.
1930                  */
1931
1932                 if (wcnt == 0) {
1933                         status = STATUS_SUCCESS;
1934                         goto wait_done;
1935                 }
1936
1937                 /*
1938                  * If this is WAITTYPE_ALL wait, and there's still
1939                  * objects that haven't been signalled, deduct the
1940                  * time that's elapsed so far from the timeout and
1941                  * wait again (or continue waiting indefinitely if
1942                  * there's no timeout).
1943                  */
1944
1945                 if (duetime != NULL) {
1946                         tv.tv_sec -= (t2.tv_sec - t1.tv_sec);
1947                         tv.tv_usec -= (t2.tv_nsec - t1.tv_nsec) / 1000;
1948                 }
1949         }
1950
1951
1952 wait_done:
1953
1954         cv_destroy(&we.we_cv);
1955
1956         for (i = 0; i < cnt; i++) {
1957                 if (whead[i].wb_object != NULL)
1958                         RemoveEntryList(&whead[i].wb_waitlist);
1959
1960         }
1961         mtx_unlock(&ntoskrnl_dispatchlock);
1962
1963         return (status);
1964 }
1965
1966 static void
1967 WRITE_REGISTER_USHORT(uint16_t *reg, uint16_t val)
1968 {
1969         bus_space_write_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1970 }
1971
1972 static uint16_t
1973 READ_REGISTER_USHORT(reg)
1974         uint16_t                *reg;
1975 {
1976         return (bus_space_read_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1977 }
1978
1979 static void
1980 WRITE_REGISTER_ULONG(reg, val)
1981         uint32_t                *reg;
1982         uint32_t                val;
1983 {
1984         bus_space_write_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1985 }
1986
1987 static uint32_t
1988 READ_REGISTER_ULONG(reg)
1989         uint32_t                *reg;
1990 {
1991         return (bus_space_read_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1992 }
1993
1994 static uint8_t
1995 READ_REGISTER_UCHAR(uint8_t *reg)
1996 {
1997         return (bus_space_read_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1998 }
1999
2000 static void
2001 WRITE_REGISTER_UCHAR(uint8_t *reg, uint8_t val)
2002 {
2003         bus_space_write_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
2004 }
2005
2006 static int64_t
2007 _allmul(a, b)
2008         int64_t                 a;
2009         int64_t                 b;
2010 {
2011         return (a * b);
2012 }
2013
2014 static int64_t
2015 _alldiv(a, b)
2016         int64_t                 a;
2017         int64_t                 b;
2018 {
2019         return (a / b);
2020 }
2021
2022 static int64_t
2023 _allrem(a, b)
2024         int64_t                 a;
2025         int64_t                 b;
2026 {
2027         return (a % b);
2028 }
2029
2030 static uint64_t
2031 _aullmul(a, b)
2032         uint64_t                a;
2033         uint64_t                b;
2034 {
2035         return (a * b);
2036 }
2037
2038 static uint64_t
2039 _aulldiv(a, b)
2040         uint64_t                a;
2041         uint64_t                b;
2042 {
2043         return (a / b);
2044 }
2045
2046 static uint64_t
2047 _aullrem(a, b)
2048         uint64_t                a;
2049         uint64_t                b;
2050 {
2051         return (a % b);
2052 }
2053
2054 static int64_t
2055 _allshl(int64_t a, uint8_t b)
2056 {
2057         return (a << b);
2058 }
2059
2060 static uint64_t
2061 _aullshl(uint64_t a, uint8_t b)
2062 {
2063         return (a << b);
2064 }
2065
2066 static int64_t
2067 _allshr(int64_t a, uint8_t b)
2068 {
2069         return (a >> b);
2070 }
2071
2072 static uint64_t
2073 _aullshr(uint64_t a, uint8_t b)
2074 {
2075         return (a >> b);
2076 }
2077
2078 static slist_entry *
2079 ntoskrnl_pushsl(head, entry)
2080         slist_header            *head;
2081         slist_entry             *entry;
2082 {
2083         slist_entry             *oldhead;
2084
2085         oldhead = head->slh_list.slh_next;
2086         entry->sl_next = head->slh_list.slh_next;
2087         head->slh_list.slh_next = entry;
2088         head->slh_list.slh_depth++;
2089         head->slh_list.slh_seq++;
2090
2091         return (oldhead);
2092 }
2093
2094 static void
2095 InitializeSListHead(head)
2096         slist_header            *head;
2097 {
2098         memset(head, 0, sizeof(*head));
2099 }
2100
2101 static slist_entry *
2102 ntoskrnl_popsl(head)
2103         slist_header            *head;
2104 {
2105         slist_entry             *first;
2106
2107         first = head->slh_list.slh_next;
2108         if (first != NULL) {
2109                 head->slh_list.slh_next = first->sl_next;
2110                 head->slh_list.slh_depth--;
2111                 head->slh_list.slh_seq++;
2112         }
2113
2114         return (first);
2115 }
2116
2117 /*
2118  * We need this to make lookaside lists work for amd64.
2119  * We pass a pointer to ExAllocatePoolWithTag() the lookaside
2120  * list structure. For amd64 to work right, this has to be a
2121  * pointer to the wrapped version of the routine, not the
2122  * original. Letting the Windows driver invoke the original
2123  * function directly will result in a convention calling
2124  * mismatch and a pretty crash. On x86, this effectively
2125  * becomes a no-op since ipt_func and ipt_wrap are the same.
2126  */
2127
2128 static funcptr
2129 ntoskrnl_findwrap(func)
2130         funcptr                 func;
2131 {
2132         image_patch_table       *patch;
2133
2134         patch = ntoskrnl_functbl;
2135         while (patch->ipt_func != NULL) {
2136                 if ((funcptr)patch->ipt_func == func)
2137                         return ((funcptr)patch->ipt_wrap);
2138                 patch++;
2139         }
2140
2141         return (NULL);
2142 }
2143
2144 static void
2145 ExInitializePagedLookasideList(paged_lookaside_list *lookaside,
2146         lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2147         uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2148 {
2149         bzero((char *)lookaside, sizeof(paged_lookaside_list));
2150
2151         if (size < sizeof(slist_entry))
2152                 lookaside->nll_l.gl_size = sizeof(slist_entry);
2153         else
2154                 lookaside->nll_l.gl_size = size;
2155         lookaside->nll_l.gl_tag = tag;
2156         if (allocfunc == NULL)
2157                 lookaside->nll_l.gl_allocfunc =
2158                     ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2159         else
2160                 lookaside->nll_l.gl_allocfunc = allocfunc;
2161
2162         if (freefunc == NULL)
2163                 lookaside->nll_l.gl_freefunc =
2164                     ntoskrnl_findwrap((funcptr)ExFreePool);
2165         else
2166                 lookaside->nll_l.gl_freefunc = freefunc;
2167
2168 #ifdef __i386__
2169         KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2170 #endif
2171
2172         lookaside->nll_l.gl_type = NonPagedPool;
2173         lookaside->nll_l.gl_depth = depth;
2174         lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2175 }
2176
2177 static void
2178 ExDeletePagedLookasideList(lookaside)
2179         paged_lookaside_list   *lookaside;
2180 {
2181         void                    *buf;
2182         void            (*freefunc)(void *);
2183
2184         freefunc = lookaside->nll_l.gl_freefunc;
2185         while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2186                 MSCALL1(freefunc, buf);
2187 }
2188
2189 static void
2190 ExInitializeNPagedLookasideList(npaged_lookaside_list *lookaside,
2191         lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2192         uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2193 {
2194         bzero((char *)lookaside, sizeof(npaged_lookaside_list));
2195
2196         if (size < sizeof(slist_entry))
2197                 lookaside->nll_l.gl_size = sizeof(slist_entry);
2198         else
2199                 lookaside->nll_l.gl_size = size;
2200         lookaside->nll_l.gl_tag = tag;
2201         if (allocfunc == NULL)
2202                 lookaside->nll_l.gl_allocfunc =
2203                     ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2204         else
2205                 lookaside->nll_l.gl_allocfunc = allocfunc;
2206
2207         if (freefunc == NULL)
2208                 lookaside->nll_l.gl_freefunc =
2209                     ntoskrnl_findwrap((funcptr)ExFreePool);
2210         else
2211                 lookaside->nll_l.gl_freefunc = freefunc;
2212
2213 #ifdef __i386__
2214         KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2215 #endif
2216
2217         lookaside->nll_l.gl_type = NonPagedPool;
2218         lookaside->nll_l.gl_depth = depth;
2219         lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2220 }
2221
2222 static void
2223 ExDeleteNPagedLookasideList(lookaside)
2224         npaged_lookaside_list   *lookaside;
2225 {
2226         void                    *buf;
2227         void            (*freefunc)(void *);
2228
2229         freefunc = lookaside->nll_l.gl_freefunc;
2230         while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2231                 MSCALL1(freefunc, buf);
2232 }
2233
2234 slist_entry *
2235 InterlockedPushEntrySList(head, entry)
2236         slist_header            *head;
2237         slist_entry             *entry;
2238 {
2239         slist_entry             *oldhead;
2240
2241         mtx_lock_spin(&ntoskrnl_interlock);
2242         oldhead = ntoskrnl_pushsl(head, entry);
2243         mtx_unlock_spin(&ntoskrnl_interlock);
2244
2245         return (oldhead);
2246 }
2247
2248 slist_entry *
2249 InterlockedPopEntrySList(head)
2250         slist_header            *head;
2251 {
2252         slist_entry             *first;
2253
2254         mtx_lock_spin(&ntoskrnl_interlock);
2255         first = ntoskrnl_popsl(head);
2256         mtx_unlock_spin(&ntoskrnl_interlock);
2257
2258         return (first);
2259 }
2260
2261 static slist_entry *
2262 ExInterlockedPushEntrySList(head, entry, lock)
2263         slist_header            *head;
2264         slist_entry             *entry;
2265         kspin_lock              *lock;
2266 {
2267         return (InterlockedPushEntrySList(head, entry));
2268 }
2269
2270 static slist_entry *
2271 ExInterlockedPopEntrySList(head, lock)
2272         slist_header            *head;
2273         kspin_lock              *lock;
2274 {
2275         return (InterlockedPopEntrySList(head));
2276 }
2277
2278 uint16_t
2279 ExQueryDepthSList(head)
2280         slist_header            *head;
2281 {
2282         uint16_t                depth;
2283
2284         mtx_lock_spin(&ntoskrnl_interlock);
2285         depth = head->slh_list.slh_depth;
2286         mtx_unlock_spin(&ntoskrnl_interlock);
2287
2288         return (depth);
2289 }
2290
2291 void
2292 KeInitializeSpinLock(lock)
2293         kspin_lock              *lock;
2294 {
2295         *lock = 0;
2296 }
2297
2298 #ifdef __i386__
2299 void
2300 KefAcquireSpinLockAtDpcLevel(lock)
2301         kspin_lock              *lock;
2302 {
2303 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2304         int                     i = 0;
2305 #endif
2306
2307         while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) {
2308                 /* sit and spin */;
2309 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2310                 i++;
2311                 if (i > 200000000)
2312                         panic("DEADLOCK!");
2313 #endif
2314         }
2315 }
2316
2317 void
2318 KefReleaseSpinLockFromDpcLevel(lock)
2319         kspin_lock              *lock;
2320 {
2321         atomic_store_rel_int((volatile u_int *)lock, 0);
2322 }
2323
2324 uint8_t
2325 KeAcquireSpinLockRaiseToDpc(kspin_lock *lock)
2326 {
2327         uint8_t                 oldirql;
2328
2329         if (KeGetCurrentIrql() > DISPATCH_LEVEL)
2330                 panic("IRQL_NOT_LESS_THAN_OR_EQUAL");
2331
2332         KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
2333         KeAcquireSpinLockAtDpcLevel(lock);
2334
2335         return (oldirql);
2336 }
2337 #else
2338 void
2339 KeAcquireSpinLockAtDpcLevel(kspin_lock *lock)
2340 {
2341         while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0)
2342                 /* sit and spin */;
2343 }
2344
2345 void
2346 KeReleaseSpinLockFromDpcLevel(kspin_lock *lock)
2347 {
2348         atomic_store_rel_int((volatile u_int *)lock, 0);
2349 }
2350 #endif /* __i386__ */
2351
2352 uintptr_t
2353 InterlockedExchange(dst, val)
2354         volatile uint32_t       *dst;
2355         uintptr_t               val;
2356 {
2357         uintptr_t               r;
2358
2359         mtx_lock_spin(&ntoskrnl_interlock);
2360         r = *dst;
2361         *dst = val;
2362         mtx_unlock_spin(&ntoskrnl_interlock);
2363
2364         return (r);
2365 }
2366
2367 static uint32_t
2368 InterlockedIncrement(addend)
2369         volatile uint32_t       *addend;
2370 {
2371         atomic_add_long((volatile u_long *)addend, 1);
2372         return (*addend);
2373 }
2374
2375 static uint32_t
2376 InterlockedDecrement(addend)
2377         volatile uint32_t       *addend;
2378 {
2379         atomic_subtract_long((volatile u_long *)addend, 1);
2380         return (*addend);
2381 }
2382
2383 static void
2384 ExInterlockedAddLargeStatistic(addend, inc)
2385         uint64_t                *addend;
2386         uint32_t                inc;
2387 {
2388         mtx_lock_spin(&ntoskrnl_interlock);
2389         *addend += inc;
2390         mtx_unlock_spin(&ntoskrnl_interlock);
2391 };
2392
2393 mdl *
2394 IoAllocateMdl(void *vaddr, uint32_t len, uint8_t secondarybuf,
2395         uint8_t chargequota, irp *iopkt)
2396 {
2397         mdl                     *m;
2398         int                     zone = 0;
2399
2400         if (MmSizeOfMdl(vaddr, len) > MDL_ZONE_SIZE)
2401                 m = ExAllocatePoolWithTag(NonPagedPool,
2402                     MmSizeOfMdl(vaddr, len), 0);
2403         else {
2404                 m = uma_zalloc(mdl_zone, M_NOWAIT | M_ZERO);
2405                 zone++;
2406         }
2407
2408         if (m == NULL)
2409                 return (NULL);
2410
2411         MmInitializeMdl(m, vaddr, len);
2412
2413         /*
2414          * MmInitializMdl() clears the flags field, so we
2415          * have to set this here. If the MDL came from the
2416          * MDL UMA zone, tag it so we can release it to
2417          * the right place later.
2418          */
2419         if (zone)
2420                 m->mdl_flags = MDL_ZONE_ALLOCED;
2421
2422         if (iopkt != NULL) {
2423                 if (secondarybuf == TRUE) {
2424                         mdl                     *last;
2425                         last = iopkt->irp_mdl;
2426                         while (last->mdl_next != NULL)
2427                                 last = last->mdl_next;
2428                         last->mdl_next = m;
2429                 } else {
2430                         if (iopkt->irp_mdl != NULL)
2431                                 panic("leaking an MDL in IoAllocateMdl()");
2432                         iopkt->irp_mdl = m;
2433                 }
2434         }
2435
2436         return (m);
2437 }
2438
2439 void
2440 IoFreeMdl(m)
2441         mdl                     *m;
2442 {
2443         if (m == NULL)
2444                 return;
2445
2446         if (m->mdl_flags & MDL_ZONE_ALLOCED)
2447                 uma_zfree(mdl_zone, m);
2448         else
2449                 ExFreePool(m);
2450 }
2451
2452 static void *
2453 MmAllocateContiguousMemory(size, highest)
2454         uint32_t                size;
2455         uint64_t                highest;
2456 {
2457         void *addr;
2458         size_t pagelength = roundup(size, PAGE_SIZE);
2459
2460         addr = ExAllocatePoolWithTag(NonPagedPool, pagelength, 0);
2461
2462         return (addr);
2463 }
2464
2465 static void *
2466 MmAllocateContiguousMemorySpecifyCache(size, lowest, highest,
2467     boundary, cachetype)
2468         uint32_t                size;
2469         uint64_t                lowest;
2470         uint64_t                highest;
2471         uint64_t                boundary;
2472         enum nt_caching_type    cachetype;
2473 {
2474         vm_memattr_t            memattr;
2475         void                    *ret;
2476
2477         switch (cachetype) {
2478         case MmNonCached:
2479                 memattr = VM_MEMATTR_UNCACHEABLE;
2480                 break;
2481         case MmWriteCombined:
2482                 memattr = VM_MEMATTR_WRITE_COMBINING;
2483                 break;
2484         case MmNonCachedUnordered:
2485                 memattr = VM_MEMATTR_UNCACHEABLE;
2486                 break;
2487         case MmCached:
2488         case MmHardwareCoherentCached:
2489         case MmUSWCCached:
2490         default:
2491                 memattr = VM_MEMATTR_DEFAULT;
2492                 break;
2493         }
2494
2495         ret = (void *)kmem_alloc_contig(size, M_ZERO | M_NOWAIT, lowest,
2496             highest, PAGE_SIZE, boundary, memattr);
2497         if (ret != NULL)
2498                 malloc_type_allocated(M_DEVBUF, round_page(size));
2499         return (ret);
2500 }
2501
2502 static void
2503 MmFreeContiguousMemory(base)
2504         void                    *base;
2505 {
2506         ExFreePool(base);
2507 }
2508
2509 static void
2510 MmFreeContiguousMemorySpecifyCache(base, size, cachetype)
2511         void                    *base;
2512         uint32_t                size;
2513         enum nt_caching_type    cachetype;
2514 {
2515         contigfree(base, size, M_DEVBUF);
2516 }
2517
2518 static uint32_t
2519 MmSizeOfMdl(vaddr, len)
2520         void                    *vaddr;
2521         size_t                  len;
2522 {
2523         uint32_t                l;
2524
2525         l = sizeof(struct mdl) +
2526             (sizeof(vm_offset_t *) * SPAN_PAGES(vaddr, len));
2527
2528         return (l);
2529 }
2530
2531 /*
2532  * The Microsoft documentation says this routine fills in the
2533  * page array of an MDL with the _physical_ page addresses that
2534  * comprise the buffer, but we don't really want to do that here.
2535  * Instead, we just fill in the page array with the kernel virtual
2536  * addresses of the buffers.
2537  */
2538 void
2539 MmBuildMdlForNonPagedPool(m)
2540         mdl                     *m;
2541 {
2542         vm_offset_t             *mdl_pages;
2543         int                     pagecnt, i;
2544
2545         pagecnt = SPAN_PAGES(m->mdl_byteoffset, m->mdl_bytecount);
2546
2547         if (pagecnt > (m->mdl_size - sizeof(mdl)) / sizeof(vm_offset_t *))
2548                 panic("not enough pages in MDL to describe buffer");
2549
2550         mdl_pages = MmGetMdlPfnArray(m);
2551
2552         for (i = 0; i < pagecnt; i++)
2553                 *mdl_pages = (vm_offset_t)m->mdl_startva + (i * PAGE_SIZE);
2554
2555         m->mdl_flags |= MDL_SOURCE_IS_NONPAGED_POOL;
2556         m->mdl_mappedsystemva = MmGetMdlVirtualAddress(m);
2557 }
2558
2559 static void *
2560 MmMapLockedPages(mdl *buf, uint8_t accessmode)
2561 {
2562         buf->mdl_flags |= MDL_MAPPED_TO_SYSTEM_VA;
2563         return (MmGetMdlVirtualAddress(buf));
2564 }
2565
2566 static void *
2567 MmMapLockedPagesSpecifyCache(mdl *buf, uint8_t accessmode, uint32_t cachetype,
2568         void *vaddr, uint32_t bugcheck, uint32_t prio)
2569 {
2570         return (MmMapLockedPages(buf, accessmode));
2571 }
2572
2573 static void
2574 MmUnmapLockedPages(vaddr, buf)
2575         void                    *vaddr;
2576         mdl                     *buf;
2577 {
2578         buf->mdl_flags &= ~MDL_MAPPED_TO_SYSTEM_VA;
2579 }
2580
2581 /*
2582  * This function has a problem in that it will break if you
2583  * compile this module without PAE and try to use it on a PAE
2584  * kernel. Unfortunately, there's no way around this at the
2585  * moment. It's slightly less broken that using pmap_kextract().
2586  * You'd think the virtual memory subsystem would help us out
2587  * here, but it doesn't.
2588  */
2589
2590 static uint64_t
2591 MmGetPhysicalAddress(void *base)
2592 {
2593         return (pmap_extract(kernel_map->pmap, (vm_offset_t)base));
2594 }
2595
2596 void *
2597 MmGetSystemRoutineAddress(ustr)
2598         unicode_string          *ustr;
2599 {
2600         ansi_string             astr;
2601
2602         if (RtlUnicodeStringToAnsiString(&astr, ustr, TRUE))
2603                 return (NULL);
2604         return (ndis_get_routine_address(ntoskrnl_functbl, astr.as_buf));
2605 }
2606
2607 uint8_t
2608 MmIsAddressValid(vaddr)
2609         void                    *vaddr;
2610 {
2611         if (pmap_extract(kernel_map->pmap, (vm_offset_t)vaddr))
2612                 return (TRUE);
2613
2614         return (FALSE);
2615 }
2616
2617 void *
2618 MmMapIoSpace(paddr, len, cachetype)
2619         uint64_t                paddr;
2620         uint32_t                len;
2621         uint32_t                cachetype;
2622 {
2623         devclass_t              nexus_class;
2624         device_t                *nexus_devs, devp;
2625         int                     nexus_count = 0;
2626         device_t                matching_dev = NULL;
2627         struct resource         *res;
2628         int                     i;
2629         vm_offset_t             v;
2630
2631         /* There will always be at least one nexus. */
2632
2633         nexus_class = devclass_find("nexus");
2634         devclass_get_devices(nexus_class, &nexus_devs, &nexus_count);
2635
2636         for (i = 0; i < nexus_count; i++) {
2637                 devp = nexus_devs[i];
2638                 matching_dev = ntoskrnl_finddev(devp, paddr, &res);
2639                 if (matching_dev)
2640                         break;
2641         }
2642
2643         free(nexus_devs, M_TEMP);
2644
2645         if (matching_dev == NULL)
2646                 return (NULL);
2647
2648         v = (vm_offset_t)rman_get_virtual(res);
2649         if (paddr > rman_get_start(res))
2650                 v += paddr - rman_get_start(res);
2651
2652         return ((void *)v);
2653 }
2654
2655 void
2656 MmUnmapIoSpace(vaddr, len)
2657         void                    *vaddr;
2658         size_t                  len;
2659 {
2660 }
2661
2662
2663 static device_t
2664 ntoskrnl_finddev(dev, paddr, res)
2665         device_t                dev;
2666         uint64_t                paddr;
2667         struct resource         **res;
2668 {
2669         device_t                *children = NULL;
2670         device_t                matching_dev;
2671         int                     childcnt;
2672         struct resource         *r;
2673         struct resource_list    *rl;
2674         struct resource_list_entry      *rle;
2675         uint32_t                flags;
2676         int                     i;
2677
2678         /* We only want devices that have been successfully probed. */
2679
2680         if (device_is_alive(dev) == FALSE)
2681                 return (NULL);
2682
2683         rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
2684         if (rl != NULL) {
2685                 STAILQ_FOREACH(rle, rl, link) {
2686                         r = rle->res;
2687
2688                         if (r == NULL)
2689                                 continue;
2690
2691                         flags = rman_get_flags(r);
2692
2693                         if (rle->type == SYS_RES_MEMORY &&
2694                             paddr >= rman_get_start(r) &&
2695                             paddr <= rman_get_end(r)) {
2696                                 if (!(flags & RF_ACTIVE))
2697                                         bus_activate_resource(dev,
2698                                             SYS_RES_MEMORY, 0, r);
2699                                 *res = r;
2700                                 return (dev);
2701                         }
2702                 }
2703         }
2704
2705         /*
2706          * If this device has children, do another
2707          * level of recursion to inspect them.
2708          */
2709
2710         device_get_children(dev, &children, &childcnt);
2711
2712         for (i = 0; i < childcnt; i++) {
2713                 matching_dev = ntoskrnl_finddev(children[i], paddr, res);
2714                 if (matching_dev != NULL) {
2715                         free(children, M_TEMP);
2716                         return (matching_dev);
2717                 }
2718         }
2719
2720
2721         /* Won't somebody please think of the children! */
2722
2723         if (children != NULL)
2724                 free(children, M_TEMP);
2725
2726         return (NULL);
2727 }
2728
2729 /*
2730  * Workitems are unlike DPCs, in that they run in a user-mode thread
2731  * context rather than at DISPATCH_LEVEL in kernel context. In our
2732  * case we run them in kernel context anyway.
2733  */
2734 static void
2735 ntoskrnl_workitem_thread(arg)
2736         void                    *arg;
2737 {
2738         kdpc_queue              *kq;
2739         list_entry              *l;
2740         io_workitem             *iw;
2741         uint8_t                 irql;
2742
2743         kq = arg;
2744
2745         InitializeListHead(&kq->kq_disp);
2746         kq->kq_td = curthread;
2747         kq->kq_exit = 0;
2748         KeInitializeSpinLock(&kq->kq_lock);
2749         KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
2750
2751         while (1) {
2752                 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
2753
2754                 KeAcquireSpinLock(&kq->kq_lock, &irql);
2755
2756                 if (kq->kq_exit) {
2757                         kq->kq_exit = 0;
2758                         KeReleaseSpinLock(&kq->kq_lock, irql);
2759                         break;
2760                 }
2761
2762                 while (!IsListEmpty(&kq->kq_disp)) {
2763                         l = RemoveHeadList(&kq->kq_disp);
2764                         iw = CONTAINING_RECORD(l,
2765                             io_workitem, iw_listentry);
2766                         InitializeListHead((&iw->iw_listentry));
2767                         if (iw->iw_func == NULL)
2768                                 continue;
2769                         KeReleaseSpinLock(&kq->kq_lock, irql);
2770                         MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx);
2771                         KeAcquireSpinLock(&kq->kq_lock, &irql);
2772                 }
2773
2774                 KeReleaseSpinLock(&kq->kq_lock, irql);
2775         }
2776
2777         kproc_exit(0);
2778         return; /* notreached */
2779 }
2780
2781 static ndis_status
2782 RtlCharToInteger(src, base, val)
2783         const char              *src;
2784         uint32_t                base;
2785         uint32_t                *val;
2786 {
2787         int negative = 0;
2788         uint32_t res;
2789
2790         if (!src || !val)
2791                 return (STATUS_ACCESS_VIOLATION);
2792         while (*src != '\0' && *src <= ' ')
2793                 src++;
2794         if (*src == '+')
2795                 src++;
2796         else if (*src == '-') {
2797                 src++;
2798                 negative = 1;
2799         }
2800         if (base == 0) {
2801                 base = 10;
2802                 if (*src == '0') {
2803                         src++;
2804                         if (*src == 'b') {
2805                                 base = 2;
2806                                 src++;
2807                         } else if (*src == 'o') {
2808                                 base = 8;
2809                                 src++;
2810                         } else if (*src == 'x') {
2811                                 base = 16;
2812                                 src++;
2813                         }
2814                 }
2815         } else if (!(base == 2 || base == 8 || base == 10 || base == 16))
2816                 return (STATUS_INVALID_PARAMETER);
2817
2818         for (res = 0; *src; src++) {
2819                 int v;
2820                 if (isdigit(*src))
2821                         v = *src - '0';
2822                 else if (isxdigit(*src))
2823                         v = tolower(*src) - 'a' + 10;
2824                 else
2825                         v = base;
2826                 if (v >= base)
2827                         return (STATUS_INVALID_PARAMETER);
2828                 res = res * base + v;
2829         }
2830         *val = negative ? -res : res;
2831         return (STATUS_SUCCESS);
2832 }
2833
2834 static void
2835 ntoskrnl_destroy_workitem_threads(void)
2836 {
2837         kdpc_queue              *kq;
2838         int                     i;
2839
2840         for (i = 0; i < WORKITEM_THREADS; i++) {
2841                 kq = wq_queues + i;
2842                 kq->kq_exit = 1;
2843                 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2844                 while (kq->kq_exit)
2845                         tsleep(kq->kq_td->td_proc, PWAIT, "waitiw", hz/10);
2846         }
2847 }
2848
2849 io_workitem *
2850 IoAllocateWorkItem(dobj)
2851         device_object           *dobj;
2852 {
2853         io_workitem             *iw;
2854
2855         iw = uma_zalloc(iw_zone, M_NOWAIT);
2856         if (iw == NULL)
2857                 return (NULL);
2858
2859         InitializeListHead(&iw->iw_listentry);
2860         iw->iw_dobj = dobj;
2861
2862         mtx_lock(&ntoskrnl_dispatchlock);
2863         iw->iw_idx = wq_idx;
2864         WORKIDX_INC(wq_idx);
2865         mtx_unlock(&ntoskrnl_dispatchlock);
2866
2867         return (iw);
2868 }
2869
2870 void
2871 IoFreeWorkItem(iw)
2872         io_workitem             *iw;
2873 {
2874         uma_zfree(iw_zone, iw);
2875 }
2876
2877 void
2878 IoQueueWorkItem(iw, iw_func, qtype, ctx)
2879         io_workitem             *iw;
2880         io_workitem_func        iw_func;
2881         uint32_t                qtype;
2882         void                    *ctx;
2883 {
2884         kdpc_queue              *kq;
2885         list_entry              *l;
2886         io_workitem             *cur;
2887         uint8_t                 irql;
2888
2889         kq = wq_queues + iw->iw_idx;
2890
2891         KeAcquireSpinLock(&kq->kq_lock, &irql);
2892
2893         /*
2894          * Traverse the list and make sure this workitem hasn't
2895          * already been inserted. Queuing the same workitem
2896          * twice will hose the list but good.
2897          */
2898
2899         l = kq->kq_disp.nle_flink;
2900         while (l != &kq->kq_disp) {
2901                 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2902                 if (cur == iw) {
2903                         /* Already queued -- do nothing. */
2904                         KeReleaseSpinLock(&kq->kq_lock, irql);
2905                         return;
2906                 }
2907                 l = l->nle_flink;
2908         }
2909
2910         iw->iw_func = iw_func;
2911         iw->iw_ctx = ctx;
2912
2913         InsertTailList((&kq->kq_disp), (&iw->iw_listentry));
2914         KeReleaseSpinLock(&kq->kq_lock, irql);
2915
2916         KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2917 }
2918
2919 static void
2920 ntoskrnl_workitem(dobj, arg)
2921         device_object           *dobj;
2922         void                    *arg;
2923 {
2924         io_workitem             *iw;
2925         work_queue_item         *w;
2926         work_item_func          f;
2927
2928         iw = arg;
2929         w = (work_queue_item *)dobj;
2930         f = (work_item_func)w->wqi_func;
2931         uma_zfree(iw_zone, iw);
2932         MSCALL2(f, w, w->wqi_ctx);
2933 }
2934
2935 /*
2936  * The ExQueueWorkItem() API is deprecated in Windows XP. Microsoft
2937  * warns that it's unsafe and to use IoQueueWorkItem() instead. The
2938  * problem with ExQueueWorkItem() is that it can't guard against
2939  * the condition where a driver submits a job to the work queue and
2940  * is then unloaded before the job is able to run. IoQueueWorkItem()
2941  * acquires a reference to the device's device_object via the
2942  * object manager and retains it until after the job has completed,
2943  * which prevents the driver from being unloaded before the job
2944  * runs. (We don't currently support this behavior, though hopefully
2945  * that will change once the object manager API is fleshed out a bit.)
2946  *
2947  * Having said all that, the ExQueueWorkItem() API remains, because
2948  * there are still other parts of Windows that use it, including
2949  * NDIS itself: NdisScheduleWorkItem() calls ExQueueWorkItem().
2950  * We fake up the ExQueueWorkItem() API on top of our implementation
2951  * of IoQueueWorkItem(). Workitem thread #3 is reserved exclusively
2952  * for ExQueueWorkItem() jobs, and we pass a pointer to the work
2953  * queue item (provided by the caller) in to IoAllocateWorkItem()
2954  * instead of the device_object. We need to save this pointer so
2955  * we can apply a sanity check: as with the DPC queue and other
2956  * workitem queues, we can't allow the same work queue item to
2957  * be queued twice. If it's already pending, we silently return
2958  */
2959
2960 void
2961 ExQueueWorkItem(w, qtype)
2962         work_queue_item         *w;
2963         uint32_t                qtype;
2964 {
2965         io_workitem             *iw;
2966         io_workitem_func        iwf;
2967         kdpc_queue              *kq;
2968         list_entry              *l;
2969         io_workitem             *cur;
2970         uint8_t                 irql;
2971
2972
2973         /*
2974          * We need to do a special sanity test to make sure
2975          * the ExQueueWorkItem() API isn't used to queue
2976          * the same workitem twice. Rather than checking the
2977          * io_workitem pointer itself, we test the attached
2978          * device object, which is really a pointer to the
2979          * legacy work queue item structure.
2980          */
2981
2982         kq = wq_queues + WORKITEM_LEGACY_THREAD;
2983         KeAcquireSpinLock(&kq->kq_lock, &irql);
2984         l = kq->kq_disp.nle_flink;
2985         while (l != &kq->kq_disp) {
2986                 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2987                 if (cur->iw_dobj == (device_object *)w) {
2988                         /* Already queued -- do nothing. */
2989                         KeReleaseSpinLock(&kq->kq_lock, irql);
2990                         return;
2991                 }
2992                 l = l->nle_flink;
2993         }
2994         KeReleaseSpinLock(&kq->kq_lock, irql);
2995
2996         iw = IoAllocateWorkItem((device_object *)w);
2997         if (iw == NULL)
2998                 return;
2999
3000         iw->iw_idx = WORKITEM_LEGACY_THREAD;
3001         iwf = (io_workitem_func)ntoskrnl_findwrap((funcptr)ntoskrnl_workitem);
3002         IoQueueWorkItem(iw, iwf, qtype, iw);
3003 }
3004
3005 static void
3006 RtlZeroMemory(dst, len)
3007         void                    *dst;
3008         size_t                  len;
3009 {
3010         bzero(dst, len);
3011 }
3012
3013 static void
3014 RtlSecureZeroMemory(dst, len)
3015         void                    *dst;
3016         size_t                  len;
3017 {
3018         memset(dst, 0, len);
3019 }
3020
3021 static void
3022 RtlFillMemory(void *dst, size_t len, uint8_t c)
3023 {
3024         memset(dst, c, len);
3025 }
3026
3027 static void
3028 RtlMoveMemory(dst, src, len)
3029         void                    *dst;
3030         const void              *src;
3031         size_t                  len;
3032 {
3033         memmove(dst, src, len);
3034 }
3035
3036 static void
3037 RtlCopyMemory(dst, src, len)
3038         void                    *dst;
3039         const void              *src;
3040         size_t                  len;
3041 {
3042         bcopy(src, dst, len);
3043 }
3044
3045 static size_t
3046 RtlCompareMemory(s1, s2, len)
3047         const void              *s1;
3048         const void              *s2;
3049         size_t                  len;
3050 {
3051         size_t                  i;
3052         uint8_t                 *m1, *m2;
3053
3054         m1 = __DECONST(char *, s1);
3055         m2 = __DECONST(char *, s2);
3056
3057         for (i = 0; i < len && m1[i] == m2[i]; i++);
3058         return (i);
3059 }
3060
3061 void
3062 RtlInitAnsiString(dst, src)
3063         ansi_string             *dst;
3064         char                    *src;
3065 {
3066         ansi_string             *a;
3067
3068         a = dst;
3069         if (a == NULL)
3070                 return;
3071         if (src == NULL) {
3072                 a->as_len = a->as_maxlen = 0;
3073                 a->as_buf = NULL;
3074         } else {
3075                 a->as_buf = src;
3076                 a->as_len = a->as_maxlen = strlen(src);
3077         }
3078 }
3079
3080 void
3081 RtlInitUnicodeString(dst, src)
3082         unicode_string          *dst;
3083         uint16_t                *src;
3084 {
3085         unicode_string          *u;
3086         int                     i;
3087
3088         u = dst;
3089         if (u == NULL)
3090                 return;
3091         if (src == NULL) {
3092                 u->us_len = u->us_maxlen = 0;
3093                 u->us_buf = NULL;
3094         } else {
3095                 i = 0;
3096                 while(src[i] != 0)
3097                         i++;
3098                 u->us_buf = src;
3099                 u->us_len = u->us_maxlen = i * 2;
3100         }
3101 }
3102
3103 ndis_status
3104 RtlUnicodeStringToInteger(ustr, base, val)
3105         unicode_string          *ustr;
3106         uint32_t                base;
3107         uint32_t                *val;
3108 {
3109         uint16_t                *uchr;
3110         int                     len, neg = 0;
3111         char                    abuf[64];
3112         char                    *astr;
3113
3114         uchr = ustr->us_buf;
3115         len = ustr->us_len;
3116         bzero(abuf, sizeof(abuf));
3117
3118         if ((char)((*uchr) & 0xFF) == '-') {
3119                 neg = 1;
3120                 uchr++;
3121                 len -= 2;
3122         } else if ((char)((*uchr) & 0xFF) == '+') {
3123                 neg = 0;
3124                 uchr++;
3125                 len -= 2;
3126         }
3127
3128         if (base == 0) {
3129                 if ((char)((*uchr) & 0xFF) == 'b') {
3130                         base = 2;
3131                         uchr++;
3132                         len -= 2;
3133                 } else if ((char)((*uchr) & 0xFF) == 'o') {
3134                         base = 8;
3135                         uchr++;
3136                         len -= 2;
3137                 } else if ((char)((*uchr) & 0xFF) == 'x') {
3138                         base = 16;
3139                         uchr++;
3140                         len -= 2;
3141                 } else
3142                         base = 10;
3143         }
3144
3145         astr = abuf;
3146         if (neg) {
3147                 strcpy(astr, "-");
3148                 astr++;
3149         }
3150
3151         ntoskrnl_unicode_to_ascii(uchr, astr, len);
3152         *val = strtoul(abuf, NULL, base);
3153
3154         return (STATUS_SUCCESS);
3155 }
3156
3157 void
3158 RtlFreeUnicodeString(ustr)
3159         unicode_string          *ustr;
3160 {
3161         if (ustr->us_buf == NULL)
3162                 return;
3163         ExFreePool(ustr->us_buf);
3164         ustr->us_buf = NULL;
3165 }
3166
3167 void
3168 RtlFreeAnsiString(astr)
3169         ansi_string             *astr;
3170 {
3171         if (astr->as_buf == NULL)
3172                 return;
3173         ExFreePool(astr->as_buf);
3174         astr->as_buf = NULL;
3175 }
3176
3177 static int
3178 atoi(str)
3179         const char              *str;
3180 {
3181         return (int)strtol(str, (char **)NULL, 10);
3182 }
3183
3184 static long
3185 atol(str)
3186         const char              *str;
3187 {
3188         return strtol(str, (char **)NULL, 10);
3189 }
3190
3191 static int
3192 rand(void)
3193 {
3194
3195         return (random());
3196 }
3197
3198 static void
3199 srand(unsigned int seed __unused)
3200 {
3201 }
3202
3203 static uint8_t
3204 IoIsWdmVersionAvailable(uint8_t major, uint8_t minor)
3205 {
3206         if (major == WDM_MAJOR && minor == WDM_MINOR_WINXP)
3207                 return (TRUE);
3208         return (FALSE);
3209 }
3210
3211 static int32_t
3212 IoOpenDeviceRegistryKey(struct device_object *devobj, uint32_t type,
3213     uint32_t mask, void **key)
3214 {
3215         return (NDIS_STATUS_INVALID_DEVICE_REQUEST);
3216 }
3217
3218 static ndis_status
3219 IoGetDeviceObjectPointer(name, reqaccess, fileobj, devobj)
3220         unicode_string          *name;
3221         uint32_t                reqaccess;
3222         void                    *fileobj;
3223         device_object           *devobj;
3224 {
3225         return (STATUS_SUCCESS);
3226 }
3227
3228 static ndis_status
3229 IoGetDeviceProperty(devobj, regprop, buflen, prop, reslen)
3230         device_object           *devobj;
3231         uint32_t                regprop;
3232         uint32_t                buflen;
3233         void                    *prop;
3234         uint32_t                *reslen;
3235 {
3236         driver_object           *drv;
3237         uint16_t                **name;
3238
3239         drv = devobj->do_drvobj;
3240
3241         switch (regprop) {
3242         case DEVPROP_DRIVER_KEYNAME:
3243                 name = prop;
3244                 *name = drv->dro_drivername.us_buf;
3245                 *reslen = drv->dro_drivername.us_len;
3246                 break;
3247         default:
3248                 return (STATUS_INVALID_PARAMETER_2);
3249                 break;
3250         }
3251
3252         return (STATUS_SUCCESS);
3253 }
3254
3255 static void
3256 KeInitializeMutex(kmutex, level)
3257         kmutant                 *kmutex;
3258         uint32_t                level;
3259 {
3260         InitializeListHead((&kmutex->km_header.dh_waitlisthead));
3261         kmutex->km_abandoned = FALSE;
3262         kmutex->km_apcdisable = 1;
3263         kmutex->km_header.dh_sigstate = 1;
3264         kmutex->km_header.dh_type = DISP_TYPE_MUTANT;
3265         kmutex->km_header.dh_size = sizeof(kmutant) / sizeof(uint32_t);
3266         kmutex->km_ownerthread = NULL;
3267 }
3268
3269 static uint32_t
3270 KeReleaseMutex(kmutant *kmutex, uint8_t kwait)
3271 {
3272         uint32_t                prevstate;
3273
3274         mtx_lock(&ntoskrnl_dispatchlock);
3275         prevstate = kmutex->km_header.dh_sigstate;
3276         if (kmutex->km_ownerthread != curthread) {
3277                 mtx_unlock(&ntoskrnl_dispatchlock);
3278                 return (STATUS_MUTANT_NOT_OWNED);
3279         }
3280
3281         kmutex->km_header.dh_sigstate++;
3282         kmutex->km_abandoned = FALSE;
3283
3284         if (kmutex->km_header.dh_sigstate == 1) {
3285                 kmutex->km_ownerthread = NULL;
3286                 ntoskrnl_waittest(&kmutex->km_header, IO_NO_INCREMENT);
3287         }
3288
3289         mtx_unlock(&ntoskrnl_dispatchlock);
3290
3291         return (prevstate);
3292 }
3293
3294 static uint32_t
3295 KeReadStateMutex(kmutex)
3296         kmutant                 *kmutex;
3297 {
3298         return (kmutex->km_header.dh_sigstate);
3299 }
3300
3301 void
3302 KeInitializeEvent(nt_kevent *kevent, uint32_t type, uint8_t state)
3303 {
3304         InitializeListHead((&kevent->k_header.dh_waitlisthead));
3305         kevent->k_header.dh_sigstate = state;
3306         if (type == EVENT_TYPE_NOTIFY)
3307                 kevent->k_header.dh_type = DISP_TYPE_NOTIFICATION_EVENT;
3308         else
3309                 kevent->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_EVENT;
3310         kevent->k_header.dh_size = sizeof(nt_kevent) / sizeof(uint32_t);
3311 }
3312
3313 uint32_t
3314 KeResetEvent(kevent)
3315         nt_kevent               *kevent;
3316 {
3317         uint32_t                prevstate;
3318
3319         mtx_lock(&ntoskrnl_dispatchlock);
3320         prevstate = kevent->k_header.dh_sigstate;
3321         kevent->k_header.dh_sigstate = FALSE;
3322         mtx_unlock(&ntoskrnl_dispatchlock);
3323
3324         return (prevstate);
3325 }
3326
3327 uint32_t
3328 KeSetEvent(nt_kevent *kevent, uint32_t increment, uint8_t kwait)
3329 {
3330         uint32_t                prevstate;
3331         wait_block              *w;
3332         nt_dispatch_header      *dh;
3333         struct thread           *td;
3334         wb_ext                  *we;
3335
3336         mtx_lock(&ntoskrnl_dispatchlock);
3337         prevstate = kevent->k_header.dh_sigstate;
3338         dh = &kevent->k_header;
3339
3340         if (IsListEmpty(&dh->dh_waitlisthead))
3341                 /*
3342                  * If there's nobody in the waitlist, just set
3343                  * the state to signalled.
3344                  */
3345                 dh->dh_sigstate = 1;
3346         else {
3347                 /*
3348                  * Get the first waiter. If this is a synchronization
3349                  * event, just wake up that one thread (don't bother
3350                  * setting the state to signalled since we're supposed
3351                  * to automatically clear synchronization events anyway).
3352                  *
3353                  * If it's a notification event, or the first
3354                  * waiter is doing a WAITTYPE_ALL wait, go through
3355                  * the full wait satisfaction process.
3356                  */
3357                 w = CONTAINING_RECORD(dh->dh_waitlisthead.nle_flink,
3358                     wait_block, wb_waitlist);
3359                 we = w->wb_ext;
3360                 td = we->we_td;
3361                 if (kevent->k_header.dh_type == DISP_TYPE_NOTIFICATION_EVENT ||
3362                     w->wb_waittype == WAITTYPE_ALL) {
3363                         if (prevstate == 0) {
3364                                 dh->dh_sigstate = 1;
3365                                 ntoskrnl_waittest(dh, increment);
3366                         }
3367                 } else {
3368                         w->wb_awakened |= TRUE;
3369                         cv_broadcastpri(&we->we_cv,
3370                             (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
3371                             w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
3372                 }
3373         }
3374
3375         mtx_unlock(&ntoskrnl_dispatchlock);
3376
3377         return (prevstate);
3378 }
3379
3380 void
3381 KeClearEvent(kevent)
3382         nt_kevent               *kevent;
3383 {
3384         kevent->k_header.dh_sigstate = FALSE;
3385 }
3386
3387 uint32_t
3388 KeReadStateEvent(kevent)
3389         nt_kevent               *kevent;
3390 {
3391         return (kevent->k_header.dh_sigstate);
3392 }
3393
3394 /*
3395  * The object manager in Windows is responsible for managing
3396  * references and access to various types of objects, including
3397  * device_objects, events, threads, timers and so on. However,
3398  * there's a difference in the way objects are handled in user
3399  * mode versus kernel mode.
3400  *
3401  * In user mode (i.e. Win32 applications), all objects are
3402  * managed by the object manager. For example, when you create
3403  * a timer or event object, you actually end up with an
3404  * object_header (for the object manager's bookkeeping
3405  * purposes) and an object body (which contains the actual object
3406  * structure, e.g. ktimer, kevent, etc...). This allows Windows
3407  * to manage resource quotas and to enforce access restrictions
3408  * on basically every kind of system object handled by the kernel.
3409  *
3410  * However, in kernel mode, you only end up using the object
3411  * manager some of the time. For example, in a driver, you create
3412  * a timer object by simply allocating the memory for a ktimer
3413  * structure and initializing it with KeInitializeTimer(). Hence,
3414  * the timer has no object_header and no reference counting or
3415  * security/resource checks are done on it. The assumption in
3416  * this case is that if you're running in kernel mode, you know
3417  * what you're doing, and you're already at an elevated privilege
3418  * anyway.
3419  *
3420  * There are some exceptions to this. The two most important ones
3421  * for our purposes are device_objects and threads. We need to use
3422  * the object manager to do reference counting on device_objects,
3423  * and for threads, you can only get a pointer to a thread's
3424  * dispatch header by using ObReferenceObjectByHandle() on the
3425  * handle returned by PsCreateSystemThread().
3426  */
3427
3428 static ndis_status
3429 ObReferenceObjectByHandle(ndis_handle handle, uint32_t reqaccess, void *otype,
3430         uint8_t accessmode, void **object, void **handleinfo)
3431 {
3432         nt_objref               *nr;
3433
3434         nr = malloc(sizeof(nt_objref), M_DEVBUF, M_NOWAIT|M_ZERO);
3435         if (nr == NULL)
3436                 return (STATUS_INSUFFICIENT_RESOURCES);
3437
3438         InitializeListHead((&nr->no_dh.dh_waitlisthead));
3439         nr->no_obj = handle;
3440         nr->no_dh.dh_type = DISP_TYPE_THREAD;
3441         nr->no_dh.dh_sigstate = 0;
3442         nr->no_dh.dh_size = (uint8_t)(sizeof(struct thread) /
3443             sizeof(uint32_t));
3444         TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link);
3445         *object = nr;
3446
3447         return (STATUS_SUCCESS);
3448 }
3449
3450 static void
3451 ObfDereferenceObject(object)
3452         void                    *object;
3453 {
3454         nt_objref               *nr;
3455
3456         nr = object;
3457         TAILQ_REMOVE(&ntoskrnl_reflist, nr, link);
3458         free(nr, M_DEVBUF);
3459 }
3460
3461 static uint32_t
3462 ZwClose(handle)
3463         ndis_handle             handle;
3464 {
3465         return (STATUS_SUCCESS);
3466 }
3467
3468 static uint32_t
3469 WmiQueryTraceInformation(traceclass, traceinfo, infolen, reqlen, buf)
3470         uint32_t                traceclass;
3471         void                    *traceinfo;
3472         uint32_t                infolen;
3473         uint32_t                reqlen;
3474         void                    *buf;
3475 {
3476         return (STATUS_NOT_FOUND);
3477 }
3478
3479 static uint32_t
3480 WmiTraceMessage(uint64_t loghandle, uint32_t messageflags,
3481         void *guid, uint16_t messagenum, ...)
3482 {
3483         return (STATUS_SUCCESS);
3484 }
3485
3486 static uint32_t
3487 IoWMIRegistrationControl(dobj, action)
3488         device_object           *dobj;
3489         uint32_t                action;
3490 {
3491         return (STATUS_SUCCESS);
3492 }
3493
3494 /*
3495  * This is here just in case the thread returns without calling
3496  * PsTerminateSystemThread().
3497  */
3498 static void
3499 ntoskrnl_thrfunc(arg)
3500         void                    *arg;
3501 {
3502         thread_context          *thrctx;
3503         uint32_t (*tfunc)(void *);
3504         void                    *tctx;
3505         uint32_t                rval;
3506
3507         thrctx = arg;
3508         tfunc = thrctx->tc_thrfunc;
3509         tctx = thrctx->tc_thrctx;
3510         free(thrctx, M_TEMP);
3511
3512         rval = MSCALL1(tfunc, tctx);
3513
3514         PsTerminateSystemThread(rval);
3515         return; /* notreached */
3516 }
3517
3518 static ndis_status
3519 PsCreateSystemThread(handle, reqaccess, objattrs, phandle,
3520         clientid, thrfunc, thrctx)
3521         ndis_handle             *handle;
3522         uint32_t                reqaccess;
3523         void                    *objattrs;
3524         ndis_handle             phandle;
3525         void                    *clientid;
3526         void                    *thrfunc;
3527         void                    *thrctx;
3528 {
3529         int                     error;
3530         thread_context          *tc;
3531         struct proc             *p;
3532
3533         tc = malloc(sizeof(thread_context), M_TEMP, M_NOWAIT);
3534         if (tc == NULL)
3535                 return (STATUS_INSUFFICIENT_RESOURCES);
3536
3537         tc->tc_thrctx = thrctx;
3538         tc->tc_thrfunc = thrfunc;
3539
3540         error = kproc_create(ntoskrnl_thrfunc, tc, &p,
3541             RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Kthread %d", ntoskrnl_kth);
3542
3543         if (error) {
3544                 free(tc, M_TEMP);
3545                 return (STATUS_INSUFFICIENT_RESOURCES);
3546         }
3547
3548         *handle = p;
3549         ntoskrnl_kth++;
3550
3551         return (STATUS_SUCCESS);
3552 }
3553
3554 /*
3555  * In Windows, the exit of a thread is an event that you're allowed
3556  * to wait on, assuming you've obtained a reference to the thread using
3557  * ObReferenceObjectByHandle(). Unfortunately, the only way we can
3558  * simulate this behavior is to register each thread we create in a
3559  * reference list, and if someone holds a reference to us, we poke
3560  * them.
3561  */
3562 static ndis_status
3563 PsTerminateSystemThread(status)
3564         ndis_status             status;
3565 {
3566         struct nt_objref        *nr;
3567
3568         mtx_lock(&ntoskrnl_dispatchlock);
3569         TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) {
3570                 if (nr->no_obj != curthread->td_proc)
3571                         continue;
3572                 nr->no_dh.dh_sigstate = 1;
3573                 ntoskrnl_waittest(&nr->no_dh, IO_NO_INCREMENT);
3574                 break;
3575         }
3576         mtx_unlock(&ntoskrnl_dispatchlock);
3577
3578         ntoskrnl_kth--;
3579
3580         kproc_exit(0);
3581         return (0);     /* notreached */
3582 }
3583
3584 static uint32_t
3585 DbgPrint(char *fmt, ...)
3586 {
3587         va_list                 ap;
3588
3589         if (bootverbose) {
3590                 va_start(ap, fmt);
3591                 vprintf(fmt, ap);
3592                 va_end(ap);
3593         }
3594
3595         return (STATUS_SUCCESS);
3596 }
3597
3598 static void
3599 DbgBreakPoint(void)
3600 {
3601
3602         kdb_enter(KDB_WHY_NDIS, "DbgBreakPoint(): breakpoint");
3603 }
3604
3605 static void
3606 KeBugCheckEx(code, param1, param2, param3, param4)
3607     uint32_t                    code;
3608     u_long                      param1;
3609     u_long                      param2;
3610     u_long                      param3;
3611     u_long                      param4;
3612 {
3613         panic("KeBugCheckEx: STOP 0x%X", code);
3614 }
3615
3616 static void
3617 ntoskrnl_timercall(arg)
3618         void                    *arg;
3619 {
3620         ktimer                  *timer;
3621         struct timeval          tv;
3622         kdpc                    *dpc;
3623
3624         mtx_lock(&ntoskrnl_dispatchlock);
3625
3626         timer = arg;
3627
3628 #ifdef NTOSKRNL_DEBUG_TIMERS
3629         ntoskrnl_timer_fires++;
3630 #endif
3631         ntoskrnl_remove_timer(timer);
3632
3633         /*
3634          * This should never happen, but complain
3635          * if it does.
3636          */
3637
3638         if (timer->k_header.dh_inserted == FALSE) {
3639                 mtx_unlock(&ntoskrnl_dispatchlock);
3640                 printf("NTOS: timer %p fired even though "
3641                     "it was canceled\n", timer);
3642                 return;
3643         }
3644
3645         /* Mark the timer as no longer being on the timer queue. */
3646
3647         timer->k_header.dh_inserted = FALSE;
3648
3649         /* Now signal the object and satisfy any waits on it. */
3650
3651         timer->k_header.dh_sigstate = 1;
3652         ntoskrnl_waittest(&timer->k_header, IO_NO_INCREMENT);
3653
3654         /*
3655          * If this is a periodic timer, re-arm it
3656          * so it will fire again. We do this before
3657          * calling any deferred procedure calls because
3658          * it's possible the DPC might cancel the timer,
3659          * in which case it would be wrong for us to
3660          * re-arm it again afterwards.
3661          */
3662
3663         if (timer->k_period) {
3664                 tv.tv_sec = 0;
3665                 tv.tv_usec = timer->k_period * 1000;
3666                 timer->k_header.dh_inserted = TRUE;
3667                 ntoskrnl_insert_timer(timer, tvtohz(&tv));
3668 #ifdef NTOSKRNL_DEBUG_TIMERS
3669                 ntoskrnl_timer_reloads++;
3670 #endif
3671         }
3672
3673         dpc = timer->k_dpc;
3674
3675         mtx_unlock(&ntoskrnl_dispatchlock);
3676
3677         /* If there's a DPC associated with the timer, queue it up. */
3678
3679         if (dpc != NULL)
3680                 KeInsertQueueDpc(dpc, NULL, NULL);
3681 }
3682
3683 #ifdef NTOSKRNL_DEBUG_TIMERS
3684 static int
3685 sysctl_show_timers(SYSCTL_HANDLER_ARGS)
3686 {
3687         int                     ret;
3688
3689         ret = 0;
3690         ntoskrnl_show_timers();
3691         return (sysctl_handle_int(oidp, &ret, 0, req));
3692 }
3693
3694 static void
3695 ntoskrnl_show_timers()
3696 {
3697         int                     i = 0;
3698         list_entry              *l;
3699
3700         mtx_lock_spin(&ntoskrnl_calllock);
3701         l = ntoskrnl_calllist.nle_flink;
3702         while(l != &ntoskrnl_calllist) {
3703                 i++;
3704                 l = l->nle_flink;
3705         }
3706         mtx_unlock_spin(&ntoskrnl_calllock);
3707
3708         printf("\n");
3709         printf("%d timers available (out of %d)\n", i, NTOSKRNL_TIMEOUTS);
3710         printf("timer sets: %qu\n", ntoskrnl_timer_sets);
3711         printf("timer reloads: %qu\n", ntoskrnl_timer_reloads);
3712         printf("timer cancels: %qu\n", ntoskrnl_timer_cancels);
3713         printf("timer fires: %qu\n", ntoskrnl_timer_fires);
3714         printf("\n");
3715 }
3716 #endif
3717
3718 /*
3719  * Must be called with dispatcher lock held.
3720  */
3721
3722 static void
3723 ntoskrnl_insert_timer(timer, ticks)
3724         ktimer                  *timer;
3725         int                     ticks;
3726 {
3727         callout_entry           *e;
3728         list_entry              *l;
3729         struct callout          *c;
3730
3731         /*
3732          * Try and allocate a timer.
3733          */
3734         mtx_lock_spin(&ntoskrnl_calllock);
3735         if (IsListEmpty(&ntoskrnl_calllist)) {
3736                 mtx_unlock_spin(&ntoskrnl_calllock);
3737 #ifdef NTOSKRNL_DEBUG_TIMERS
3738                 ntoskrnl_show_timers();
3739 #endif
3740                 panic("out of timers!");
3741         }
3742         l = RemoveHeadList(&ntoskrnl_calllist);
3743         mtx_unlock_spin(&ntoskrnl_calllock);
3744
3745         e = CONTAINING_RECORD(l, callout_entry, ce_list);
3746         c = &e->ce_callout;
3747
3748         timer->k_callout = c;
3749
3750         callout_init(c, 1);
3751         callout_reset(c, ticks, ntoskrnl_timercall, timer);
3752 }
3753
3754 static void
3755 ntoskrnl_remove_timer(timer)
3756         ktimer                  *timer;
3757 {
3758         callout_entry           *e;
3759
3760         e = (callout_entry *)timer->k_callout;
3761         callout_stop(timer->k_callout);
3762
3763         mtx_lock_spin(&ntoskrnl_calllock);
3764         InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
3765         mtx_unlock_spin(&ntoskrnl_calllock);
3766 }
3767
3768 void
3769 KeInitializeTimer(timer)
3770         ktimer                  *timer;
3771 {
3772         if (timer == NULL)
3773                 return;
3774
3775         KeInitializeTimerEx(timer,  EVENT_TYPE_NOTIFY);
3776 }
3777
3778 void
3779 KeInitializeTimerEx(timer, type)
3780         ktimer                  *timer;
3781         uint32_t                type;
3782 {
3783         if (timer == NULL)
3784                 return;
3785
3786         bzero((char *)timer, sizeof(ktimer));
3787         InitializeListHead((&timer->k_header.dh_waitlisthead));
3788         timer->k_header.dh_sigstate = FALSE;
3789         timer->k_header.dh_inserted = FALSE;
3790         if (type == EVENT_TYPE_NOTIFY)
3791                 timer->k_header.dh_type = DISP_TYPE_NOTIFICATION_TIMER;
3792         else
3793                 timer->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_TIMER;
3794         timer->k_header.dh_size = sizeof(ktimer) / sizeof(uint32_t);
3795 }
3796
3797 /*
3798  * DPC subsystem. A Windows Defered Procedure Call has the following
3799  * properties:
3800  * - It runs at DISPATCH_LEVEL.
3801  * - It can have one of 3 importance values that control when it
3802  *   runs relative to other DPCs in the queue.
3803  * - On SMP systems, it can be set to run on a specific processor.
3804  * In order to satisfy the last property, we create a DPC thread for
3805  * each CPU in the system and bind it to that CPU. Each thread
3806  * maintains three queues with different importance levels, which
3807  * will be processed in order from lowest to highest.
3808  *
3809  * In Windows, interrupt handlers run as DPCs. (Not to be confused
3810  * with ISRs, which run in interrupt context and can preempt DPCs.)
3811  * ISRs are given the highest importance so that they'll take
3812  * precedence over timers and other things.
3813  */
3814
3815 static void
3816 ntoskrnl_dpc_thread(arg)
3817         void                    *arg;
3818 {
3819         kdpc_queue              *kq;
3820         kdpc                    *d;
3821         list_entry              *l;
3822         uint8_t                 irql;
3823
3824         kq = arg;
3825
3826         InitializeListHead(&kq->kq_disp);
3827         kq->kq_td = curthread;
3828         kq->kq_exit = 0;
3829         kq->kq_running = FALSE;
3830         KeInitializeSpinLock(&kq->kq_lock);
3831         KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
3832         KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE);
3833
3834         /*
3835          * Elevate our priority. DPCs are used to run interrupt
3836          * handlers, and they should trigger as soon as possible
3837          * once scheduled by an ISR.
3838          */
3839
3840         thread_lock(curthread);
3841 #ifdef NTOSKRNL_MULTIPLE_DPCS
3842         sched_bind(curthread, kq->kq_cpu);
3843 #endif
3844         sched_prio(curthread, PRI_MIN_KERN);
3845         thread_unlock(curthread);
3846
3847         while (1) {
3848                 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
3849
3850                 KeAcquireSpinLock(&kq->kq_lock, &irql);
3851
3852                 if (kq->kq_exit) {
3853                         kq->kq_exit = 0;
3854                         KeReleaseSpinLock(&kq->kq_lock, irql);
3855                         break;
3856                 }
3857
3858                 kq->kq_running = TRUE;
3859
3860                 while (!IsListEmpty(&kq->kq_disp)) {
3861                         l = RemoveHeadList((&kq->kq_disp));
3862                         d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3863                         InitializeListHead((&d->k_dpclistentry));
3864                         KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
3865                         MSCALL4(d->k_deferedfunc, d, d->k_deferredctx,
3866                             d->k_sysarg1, d->k_sysarg2);
3867                         KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3868                 }
3869
3870                 kq->kq_running = FALSE;
3871
3872                 KeReleaseSpinLock(&kq->kq_lock, irql);
3873
3874                 KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE);
3875         }
3876
3877         kproc_exit(0);
3878         return; /* notreached */
3879 }
3880
3881 static void
3882 ntoskrnl_destroy_dpc_threads(void)
3883 {
3884         kdpc_queue              *kq;
3885         kdpc                    dpc;
3886         int                     i;
3887
3888         kq = kq_queues;
3889 #ifdef NTOSKRNL_MULTIPLE_DPCS
3890         for (i = 0; i < mp_ncpus; i++) {
3891 #else
3892         for (i = 0; i < 1; i++) {
3893 #endif
3894                 kq += i;
3895
3896                 kq->kq_exit = 1;
3897                 KeInitializeDpc(&dpc, NULL, NULL);
3898                 KeSetTargetProcessorDpc(&dpc, i);
3899                 KeInsertQueueDpc(&dpc, NULL, NULL);
3900                 while (kq->kq_exit)
3901                         tsleep(kq->kq_td->td_proc, PWAIT, "dpcw", hz/10);
3902         }
3903 }
3904
3905 static uint8_t
3906 ntoskrnl_insert_dpc(head, dpc)
3907         list_entry              *head;
3908         kdpc                    *dpc;
3909 {
3910         list_entry              *l;
3911         kdpc                    *d;
3912
3913         l = head->nle_flink;
3914         while (l != head) {
3915                 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3916                 if (d == dpc)
3917                         return (FALSE);
3918                 l = l->nle_flink;
3919         }
3920
3921         if (dpc->k_importance == KDPC_IMPORTANCE_LOW)
3922                 InsertTailList((head), (&dpc->k_dpclistentry));
3923         else
3924                 InsertHeadList((head), (&dpc->k_dpclistentry));
3925
3926         return (TRUE);
3927 }
3928
3929 void
3930 KeInitializeDpc(dpc, dpcfunc, dpcctx)
3931         kdpc                    *dpc;
3932         void                    *dpcfunc;
3933         void                    *dpcctx;
3934 {
3935
3936         if (dpc == NULL)
3937                 return;
3938
3939         dpc->k_deferedfunc = dpcfunc;
3940         dpc->k_deferredctx = dpcctx;
3941         dpc->k_num = KDPC_CPU_DEFAULT;
3942         dpc->k_importance = KDPC_IMPORTANCE_MEDIUM;
3943         InitializeListHead((&dpc->k_dpclistentry));
3944 }
3945
3946 uint8_t
3947 KeInsertQueueDpc(dpc, sysarg1, sysarg2)
3948         kdpc                    *dpc;
3949         void                    *sysarg1;
3950         void                    *sysarg2;
3951 {
3952         kdpc_queue              *kq;
3953         uint8_t                 r;
3954         uint8_t                 irql;
3955
3956         if (dpc == NULL)
3957                 return (FALSE);
3958
3959         kq = kq_queues;
3960
3961 #ifdef NTOSKRNL_MULTIPLE_DPCS
3962         KeRaiseIrql(DISPATCH_LEVEL, &irql);
3963
3964         /*
3965          * By default, the DPC is queued to run on the same CPU
3966          * that scheduled it.
3967          */
3968
3969         if (dpc->k_num == KDPC_CPU_DEFAULT)
3970                 kq += curthread->td_oncpu;
3971         else
3972                 kq += dpc->k_num;
3973         KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3974 #else
3975         KeAcquireSpinLock(&kq->kq_lock, &irql);
3976 #endif
3977
3978         r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc);
3979         if (r == TRUE) {
3980                 dpc->k_sysarg1 = sysarg1;
3981                 dpc->k_sysarg2 = sysarg2;
3982         }
3983         KeReleaseSpinLock(&kq->kq_lock, irql);
3984
3985         if (r == FALSE)
3986                 return (r);
3987
3988         KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
3989
3990         return (r);
3991 }
3992
3993 uint8_t
3994 KeRemoveQueueDpc(dpc)
3995         kdpc                    *dpc;
3996 {
3997         kdpc_queue              *kq;
3998         uint8_t                 irql;
3999
4000         if (dpc == NULL)
4001                 return (FALSE);
4002
4003 #ifdef NTOSKRNL_MULTIPLE_DPCS
4004         KeRaiseIrql(DISPATCH_LEVEL, &irql);
4005
4006         kq = kq_queues + dpc->k_num;
4007
4008         KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
4009 #else
4010         kq = kq_queues;
4011         KeAcquireSpinLock(&kq->kq_lock, &irql);
4012 #endif
4013
4014         if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) {
4015                 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
4016                 KeLowerIrql(irql);
4017                 return (FALSE);
4018         }
4019
4020         RemoveEntryList((&dpc->k_dpclistentry));
4021         InitializeListHead((&dpc->k_dpclistentry));
4022
4023         KeReleaseSpinLock(&kq->kq_lock, irql);
4024
4025         return (TRUE);
4026 }
4027
4028 void
4029 KeSetImportanceDpc(dpc, imp)
4030         kdpc                    *dpc;
4031         uint32_t                imp;
4032 {
4033         if (imp != KDPC_IMPORTANCE_LOW &&
4034             imp != KDPC_IMPORTANCE_MEDIUM &&
4035             imp != KDPC_IMPORTANCE_HIGH)
4036                 return;
4037
4038         dpc->k_importance = (uint8_t)imp;
4039 }
4040
4041 void
4042 KeSetTargetProcessorDpc(kdpc *dpc, uint8_t cpu)
4043 {
4044         if (cpu > mp_ncpus)
4045                 return;
4046
4047         dpc->k_num = cpu;
4048 }
4049
4050 void
4051 KeFlushQueuedDpcs(void)
4052 {
4053         kdpc_queue              *kq;
4054         int                     i;
4055
4056         /*
4057          * Poke each DPC queue and wait
4058          * for them to drain.
4059          */
4060
4061 #ifdef NTOSKRNL_MULTIPLE_DPCS
4062         for (i = 0; i < mp_ncpus; i++) {
4063 #else
4064         for (i = 0; i < 1; i++) {
4065 #endif
4066                 kq = kq_queues + i;
4067                 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
4068                 KeWaitForSingleObject(&kq->kq_done, 0, 0, TRUE, NULL);
4069         }
4070 }
4071
4072 uint32_t
4073 KeGetCurrentProcessorNumber(void)
4074 {
4075         return ((uint32_t)curthread->td_oncpu);
4076 }
4077
4078 uint8_t
4079 KeSetTimerEx(timer, duetime, period, dpc)
4080         ktimer                  *timer;
4081         int64_t                 duetime;
4082         uint32_t                period;
4083         kdpc                    *dpc;
4084 {
4085         struct timeval          tv;
4086         uint64_t                curtime;
4087         uint8_t                 pending;
4088
4089         if (timer == NULL)
4090                 return (FALSE);
4091
4092         mtx_lock(&ntoskrnl_dispatchlock);
4093
4094         if (timer->k_header.dh_inserted == TRUE) {
4095                 ntoskrnl_remove_timer(timer);
4096 #ifdef NTOSKRNL_DEBUG_TIMERS
4097                 ntoskrnl_timer_cancels++;
4098 #endif
4099                 timer->k_header.dh_inserted = FALSE;
4100                 pending = TRUE;
4101         } else
4102                 pending = FALSE;
4103
4104         timer->k_duetime = duetime;
4105         timer->k_period = period;
4106         timer->k_header.dh_sigstate = FALSE;
4107         timer->k_dpc = dpc;
4108
4109         if (duetime < 0) {
4110                 tv.tv_sec = - (duetime) / 10000000;
4111                 tv.tv_usec = (- (duetime) / 10) -
4112                     (tv.tv_sec * 1000000);
4113         } else {
4114                 ntoskrnl_time(&curtime);
4115                 if (duetime < curtime)
4116                         tv.tv_sec = tv.tv_usec = 0;
4117                 else {
4118                         tv.tv_sec = ((duetime) - curtime) / 10000000;
4119                         tv.tv_usec = ((duetime) - curtime) / 10 -
4120                             (tv.tv_sec * 1000000);
4121                 }
4122         }
4123
4124         timer->k_header.dh_inserted = TRUE;
4125         ntoskrnl_insert_timer(timer, tvtohz(&tv));
4126 #ifdef NTOSKRNL_DEBUG_TIMERS
4127         ntoskrnl_timer_sets++;
4128 #endif
4129
4130         mtx_unlock(&ntoskrnl_dispatchlock);
4131
4132         return (pending);
4133 }
4134
4135 uint8_t
4136 KeSetTimer(timer, duetime, dpc)
4137         ktimer                  *timer;
4138         int64_t                 duetime;
4139         kdpc                    *dpc;
4140 {
4141         return (KeSetTimerEx(timer, duetime, 0, dpc));
4142 }
4143
4144 /*
4145  * The Windows DDK documentation seems to say that cancelling
4146  * a timer that has a DPC will result in the DPC also being
4147  * cancelled, but this isn't really the case.
4148  */
4149
4150 uint8_t
4151 KeCancelTimer(timer)
4152         ktimer                  *timer;
4153 {
4154         uint8_t                 pending;
4155
4156         if (timer == NULL)
4157                 return (FALSE);
4158
4159         mtx_lock(&ntoskrnl_dispatchlock);
4160
4161         pending = timer->k_header.dh_inserted;
4162
4163         if (timer->k_header.dh_inserted == TRUE) {
4164                 timer->k_header.dh_inserted = FALSE;
4165                 ntoskrnl_remove_timer(timer);
4166 #ifdef NTOSKRNL_DEBUG_TIMERS
4167                 ntoskrnl_timer_cancels++;
4168 #endif
4169         }
4170
4171         mtx_unlock(&ntoskrnl_dispatchlock);
4172
4173         return (pending);
4174 }
4175
4176 uint8_t
4177 KeReadStateTimer(timer)
4178         ktimer                  *timer;
4179 {
4180         return (timer->k_header.dh_sigstate);
4181 }
4182
4183 static int32_t
4184 KeDelayExecutionThread(uint8_t wait_mode, uint8_t alertable, int64_t *interval)
4185 {
4186         ktimer                  timer;
4187
4188         if (wait_mode != 0)
4189                 panic("invalid wait_mode %d", wait_mode);
4190
4191         KeInitializeTimer(&timer);
4192         KeSetTimer(&timer, *interval, NULL);
4193         KeWaitForSingleObject(&timer, 0, 0, alertable, NULL);
4194
4195         return STATUS_SUCCESS;
4196 }
4197
4198 static uint64_t
4199 KeQueryInterruptTime(void)
4200 {
4201         int ticks;
4202         struct timeval tv;
4203
4204         getmicrouptime(&tv);
4205
4206         ticks = tvtohz(&tv);
4207
4208         return ticks * howmany(10000000, hz);
4209 }
4210
4211 static struct thread *
4212 KeGetCurrentThread(void)
4213 {
4214
4215         return curthread;
4216 }
4217
4218 static int32_t
4219 KeSetPriorityThread(td, pri)
4220         struct thread   *td;
4221         int32_t         pri;
4222 {
4223         int32_t old;
4224
4225         if (td == NULL)
4226                 return LOW_REALTIME_PRIORITY;
4227
4228         if (td->td_priority <= PRI_MIN_KERN)
4229                 old = HIGH_PRIORITY;
4230         else if (td->td_priority >= PRI_MAX_KERN)
4231                 old = LOW_PRIORITY;
4232         else
4233                 old = LOW_REALTIME_PRIORITY;
4234
4235         thread_lock(td);
4236         if (pri == HIGH_PRIORITY)
4237                 sched_prio(td, PRI_MIN_KERN);
4238         if (pri == LOW_REALTIME_PRIORITY)
4239                 sched_prio(td, PRI_MIN_KERN + (PRI_MAX_KERN - PRI_MIN_KERN) / 2);
4240         if (pri == LOW_PRIORITY)
4241                 sched_prio(td, PRI_MAX_KERN);
4242         thread_unlock(td);
4243
4244         return old;
4245 }
4246
4247 static void
4248 dummy()
4249 {
4250         printf("ntoskrnl dummy called...\n");
4251 }
4252
4253
4254 image_patch_table ntoskrnl_functbl[] = {
4255         IMPORT_SFUNC(RtlZeroMemory, 2),
4256         IMPORT_SFUNC(RtlSecureZeroMemory, 2),
4257         IMPORT_SFUNC(RtlFillMemory, 3),
4258         IMPORT_SFUNC(RtlMoveMemory, 3),
4259         IMPORT_SFUNC(RtlCharToInteger, 3),
4260         IMPORT_SFUNC(RtlCopyMemory, 3),
4261         IMPORT_SFUNC(RtlCopyString, 2),
4262         IMPORT_SFUNC(RtlCompareMemory, 3),
4263         IMPORT_SFUNC(RtlEqualUnicodeString, 3),
4264         IMPORT_SFUNC(RtlCopyUnicodeString, 2),
4265         IMPORT_SFUNC(RtlUnicodeStringToAnsiString, 3),
4266         IMPORT_SFUNC(RtlAnsiStringToUnicodeString, 3),
4267         IMPORT_SFUNC(RtlInitAnsiString, 2),
4268         IMPORT_SFUNC_MAP(RtlInitString, RtlInitAnsiString, 2),
4269         IMPORT_SFUNC(RtlInitUnicodeString, 2),
4270         IMPORT_SFUNC(RtlFreeAnsiString, 1),
4271         IMPORT_SFUNC(RtlFreeUnicodeString, 1),
4272         IMPORT_SFUNC(RtlUnicodeStringToInteger, 3),
4273         IMPORT_CFUNC(sprintf, 0),
4274         IMPORT_CFUNC(vsprintf, 0),
4275         IMPORT_CFUNC_MAP(_snprintf, snprintf, 0),
4276         IMPORT_CFUNC_MAP(_vsnprintf, vsnprintf, 0),
4277         IMPORT_CFUNC(DbgPrint, 0),
4278         IMPORT_SFUNC(DbgBreakPoint, 0),
4279         IMPORT_SFUNC(KeBugCheckEx, 5),
4280         IMPORT_CFUNC(strncmp, 0),
4281         IMPORT_CFUNC(strcmp, 0),
4282         IMPORT_CFUNC_MAP(stricmp, strcasecmp, 0),
4283         IMPORT_CFUNC(strncpy, 0),
4284         IMPORT_CFUNC(strcpy, 0),
4285         IMPORT_CFUNC(strlen, 0),
4286         IMPORT_CFUNC_MAP(toupper, ntoskrnl_toupper, 0),
4287         IMPORT_CFUNC_MAP(tolower, ntoskrnl_tolower, 0),
4288         IMPORT_CFUNC_MAP(strstr, ntoskrnl_strstr, 0),
4289         IMPORT_CFUNC_MAP(strncat, ntoskrnl_strncat, 0),
4290         IMPORT_CFUNC_MAP(strchr, index, 0),
4291         IMPORT_CFUNC_MAP(strrchr, rindex, 0),
4292         IMPORT_CFUNC(memcpy, 0),
4293         IMPORT_CFUNC_MAP(memmove, ntoskrnl_memmove, 0),
4294         IMPORT_CFUNC_MAP(memset, ntoskrnl_memset, 0),
4295         IMPORT_CFUNC_MAP(memchr, ntoskrnl_memchr, 0),
4296         IMPORT_SFUNC(IoAllocateDriverObjectExtension, 4),
4297         IMPORT_SFUNC(IoGetDriverObjectExtension, 2),
4298         IMPORT_FFUNC(IofCallDriver, 2),
4299         IMPORT_FFUNC(IofCompleteRequest, 2),
4300         IMPORT_SFUNC(IoAcquireCancelSpinLock, 1),
4301         IMPORT_SFUNC(IoReleaseCancelSpinLock, 1),
4302         IMPORT_SFUNC(IoCancelIrp, 1),
4303         IMPORT_SFUNC(IoConnectInterrupt, 11),
4304         IMPORT_SFUNC(IoDisconnectInterrupt, 1),
4305         IMPORT_SFUNC(IoCreateDevice, 7),
4306         IMPORT_SFUNC(IoDeleteDevice, 1),
4307         IMPORT_SFUNC(IoGetAttachedDevice, 1),
4308         IMPORT_SFUNC(IoAttachDeviceToDeviceStack, 2),
4309         IMPORT_SFUNC(IoDetachDevice, 1),
4310         IMPORT_SFUNC(IoBuildSynchronousFsdRequest, 7),
4311         IMPORT_SFUNC(IoBuildAsynchronousFsdRequest, 6),
4312         IMPORT_SFUNC(IoBuildDeviceIoControlRequest, 9),
4313         IMPORT_SFUNC(IoAllocateIrp, 2),
4314         IMPORT_SFUNC(IoReuseIrp, 2),
4315         IMPORT_SFUNC(IoMakeAssociatedIrp, 2),
4316         IMPORT_SFUNC(IoFreeIrp, 1),
4317         IMPORT_SFUNC(IoInitializeIrp, 3),
4318         IMPORT_SFUNC(KeAcquireInterruptSpinLock, 1),
4319         IMPORT_SFUNC(KeReleaseInterruptSpinLock, 2),
4320         IMPORT_SFUNC(KeSynchronizeExecution, 3),
4321         IMPORT_SFUNC(KeWaitForSingleObject, 5),
4322         IMPORT_SFUNC(KeWaitForMultipleObjects, 8),
4323         IMPORT_SFUNC(_allmul, 4),
4324         IMPORT_SFUNC(_alldiv, 4),
4325         IMPORT_SFUNC(_allrem, 4),
4326         IMPORT_RFUNC(_allshr, 0),
4327         IMPORT_RFUNC(_allshl, 0),
4328         IMPORT_SFUNC(_aullmul, 4),
4329         IMPORT_SFUNC(_aulldiv, 4),
4330         IMPORT_SFUNC(_aullrem, 4),
4331         IMPORT_RFUNC(_aullshr, 0),
4332         IMPORT_RFUNC(_aullshl, 0),
4333         IMPORT_CFUNC(atoi, 0),
4334         IMPORT_CFUNC(atol, 0),
4335         IMPORT_CFUNC(rand, 0),
4336         IMPORT_CFUNC(srand, 0),
4337         IMPORT_SFUNC(WRITE_REGISTER_USHORT, 2),
4338         IMPORT_SFUNC(READ_REGISTER_USHORT, 1),
4339         IMPORT_SFUNC(WRITE_REGISTER_ULONG, 2),
4340         IMPORT_SFUNC(READ_REGISTER_ULONG, 1),
4341         IMPORT_SFUNC(READ_REGISTER_UCHAR, 1),
4342         IMPORT_SFUNC(WRITE_REGISTER_UCHAR, 2),
4343         IMPORT_SFUNC(ExInitializePagedLookasideList, 7),
4344         IMPORT_SFUNC(ExDeletePagedLookasideList, 1),
4345         IMPORT_SFUNC(ExInitializeNPagedLookasideList, 7),
4346         IMPORT_SFUNC(ExDeleteNPagedLookasideList, 1),
4347         IMPORT_FFUNC(InterlockedPopEntrySList, 1),
4348         IMPORT_FFUNC(InitializeSListHead, 1),
4349         IMPORT_FFUNC(InterlockedPushEntrySList, 2),
4350         IMPORT_SFUNC(ExQueryDepthSList, 1),
4351         IMPORT_FFUNC_MAP(ExpInterlockedPopEntrySList,
4352                 InterlockedPopEntrySList, 1),
4353         IMPORT_FFUNC_MAP(ExpInterlockedPushEntrySList,
4354                 InterlockedPushEntrySList, 2),
4355         IMPORT_FFUNC(ExInterlockedPopEntrySList, 2),
4356         IMPORT_FFUNC(ExInterlockedPushEntrySList, 3),
4357         IMPORT_SFUNC(ExAllocatePoolWithTag, 3),
4358         IMPORT_SFUNC(ExFreePoolWithTag, 2),
4359         IMPORT_SFUNC(ExFreePool, 1),
4360 #ifdef __i386__
4361         IMPORT_FFUNC(KefAcquireSpinLockAtDpcLevel, 1),
4362         IMPORT_FFUNC(KefReleaseSpinLockFromDpcLevel,1),
4363         IMPORT_FFUNC(KeAcquireSpinLockRaiseToDpc, 1),
4364 #else
4365         /*
4366          * For AMD64, we can get away with just mapping
4367          * KeAcquireSpinLockRaiseToDpc() directly to KfAcquireSpinLock()
4368          * because the calling conventions end up being the same.
4369          * On i386, we have to be careful because KfAcquireSpinLock()
4370          * is _fastcall but KeAcquireSpinLockRaiseToDpc() isn't.
4371          */
4372         IMPORT_SFUNC(KeAcquireSpinLockAtDpcLevel, 1),
4373         IMPORT_SFUNC(KeReleaseSpinLockFromDpcLevel, 1),
4374         IMPORT_SFUNC_MAP(KeAcquireSpinLockRaiseToDpc, KfAcquireSpinLock, 1),
4375 #endif
4376         IMPORT_SFUNC_MAP(KeReleaseSpinLock, KfReleaseSpinLock, 1),
4377         IMPORT_FFUNC(InterlockedIncrement, 1),
4378         IMPORT_FFUNC(InterlockedDecrement, 1),
4379         IMPORT_FFUNC(InterlockedExchange, 2),
4380         IMPORT_FFUNC(ExInterlockedAddLargeStatistic, 2),
4381         IMPORT_SFUNC(IoAllocateMdl, 5),
4382         IMPORT_SFUNC(IoFreeMdl, 1),
4383         IMPORT_SFUNC(MmAllocateContiguousMemory, 2 + 1),
4384         IMPORT_SFUNC(MmAllocateContiguousMemorySpecifyCache, 5 + 3),
4385         IMPORT_SFUNC(MmFreeContiguousMemory, 1),
4386         IMPORT_SFUNC(MmFreeContiguousMemorySpecifyCache, 3),
4387         IMPORT_SFUNC(MmSizeOfMdl, 1),
4388         IMPORT_SFUNC(MmMapLockedPages, 2),
4389         IMPORT_SFUNC(MmMapLockedPagesSpecifyCache, 6),
4390         IMPORT_SFUNC(MmUnmapLockedPages, 2),
4391         IMPORT_SFUNC(MmBuildMdlForNonPagedPool, 1),
4392         IMPORT_SFUNC(MmGetPhysicalAddress, 1),
4393         IMPORT_SFUNC(MmGetSystemRoutineAddress, 1),
4394         IMPORT_SFUNC(MmIsAddressValid, 1),
4395         IMPORT_SFUNC(MmMapIoSpace, 3 + 1),
4396         IMPORT_SFUNC(MmUnmapIoSpace, 2),
4397         IMPORT_SFUNC(KeInitializeSpinLock, 1),
4398         IMPORT_SFUNC(IoIsWdmVersionAvailable, 2),
4399         IMPORT_SFUNC(IoOpenDeviceRegistryKey, 4),
4400         IMPORT_SFUNC(IoGetDeviceObjectPointer, 4),
4401         IMPORT_SFUNC(IoGetDeviceProperty, 5),
4402         IMPORT_SFUNC(IoAllocateWorkItem, 1),
4403         IMPORT_SFUNC(IoFreeWorkItem, 1),
4404         IMPORT_SFUNC(IoQueueWorkItem, 4),
4405         IMPORT_SFUNC(ExQueueWorkItem, 2),
4406         IMPORT_SFUNC(ntoskrnl_workitem, 2),
4407         IMPORT_SFUNC(KeInitializeMutex, 2),
4408         IMPORT_SFUNC(KeReleaseMutex, 2),
4409         IMPORT_SFUNC(KeReadStateMutex, 1),
4410         IMPORT_SFUNC(KeInitializeEvent, 3),
4411         IMPORT_SFUNC(KeSetEvent, 3),
4412         IMPORT_SFUNC(KeResetEvent, 1),
4413         IMPORT_SFUNC(KeClearEvent, 1),
4414         IMPORT_SFUNC(KeReadStateEvent, 1),
4415         IMPORT_SFUNC(KeInitializeTimer, 1),
4416         IMPORT_SFUNC(KeInitializeTimerEx, 2),
4417         IMPORT_SFUNC(KeSetTimer, 3),
4418         IMPORT_SFUNC(KeSetTimerEx, 4),
4419         IMPORT_SFUNC(KeCancelTimer, 1),
4420         IMPORT_SFUNC(KeReadStateTimer, 1),
4421         IMPORT_SFUNC(KeInitializeDpc, 3),
4422         IMPORT_SFUNC(KeInsertQueueDpc, 3),
4423         IMPORT_SFUNC(KeRemoveQueueDpc, 1),
4424         IMPORT_SFUNC(KeSetImportanceDpc, 2),
4425         IMPORT_SFUNC(KeSetTargetProcessorDpc, 2),
4426         IMPORT_SFUNC(KeFlushQueuedDpcs, 0),
4427         IMPORT_SFUNC(KeGetCurrentProcessorNumber, 1),
4428         IMPORT_SFUNC(ObReferenceObjectByHandle, 6),
4429         IMPORT_FFUNC(ObfDereferenceObject, 1),
4430         IMPORT_SFUNC(ZwClose, 1),
4431         IMPORT_SFUNC(PsCreateSystemThread, 7),
4432         IMPORT_SFUNC(PsTerminateSystemThread, 1),
4433         IMPORT_SFUNC(IoWMIRegistrationControl, 2),
4434         IMPORT_SFUNC(WmiQueryTraceInformation, 5),
4435         IMPORT_CFUNC(WmiTraceMessage, 0),
4436         IMPORT_SFUNC(KeQuerySystemTime, 1),
4437         IMPORT_CFUNC(KeTickCount, 0),
4438         IMPORT_SFUNC(KeDelayExecutionThread, 3),
4439         IMPORT_SFUNC(KeQueryInterruptTime, 0),
4440         IMPORT_SFUNC(KeGetCurrentThread, 0),
4441         IMPORT_SFUNC(KeSetPriorityThread, 2),
4442
4443         /*
4444          * This last entry is a catch-all for any function we haven't
4445          * implemented yet. The PE import list patching routine will
4446          * use it for any function that doesn't have an explicit match
4447          * in this table.
4448          */
4449
4450         { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL },
4451
4452         /* End of list. */
4453
4454         { NULL, NULL, NULL }
4455 };