sys/compat/ndis/subr_ntoskrnl.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-4-Clause
   3  *
   4  * Copyright (c) 2003
   5  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. All advertising materials mentioning features or use of this software
  16  *    must display the following acknowledgement:
  17  *      This product includes software developed by Bill Paul.
  18  * 4. Neither the name of the author nor the names of any co-contributors
  19  *    may be used to endorse or promote products derived from this software
  20  *    without specific prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  26  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  32  * THE POSSIBILITY OF SUCH DAMAGE.
  33  */
  34
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include <sys/ctype.h>
  39 #include <sys/unistd.h>
  40 #include <sys/param.h>
  41 #include <sys/types.h>
  42 #include <sys/errno.h>
  43 #include <sys/systm.h>
  44 #include <sys/malloc.h>
  45 #include <sys/lock.h>
  46 #include <sys/mutex.h>
  47
  48 #include <sys/callout.h>
  49 #include <sys/kdb.h>
  50 #include <sys/kernel.h>
  51 #include <sys/proc.h>
  52 #include <sys/condvar.h>
  53 #include <sys/kthread.h>
  54 #include <sys/module.h>
  55 #include <sys/smp.h>
  56 #include <sys/sched.h>
  57 #include <sys/sysctl.h>
  58
  59 #include <machine/atomic.h>
  60 #include <machine/bus.h>
  61 #include <machine/stdarg.h>
  62 #include <machine/resource.h>
  63
  64 #include <sys/bus.h>
  65 #include <sys/rman.h>
  66
  67 #include <vm/vm.h>
  68 #include <vm/vm_param.h>
  69 #include <vm/pmap.h>
  70 #include <vm/uma.h>
  71 #include <vm/vm_kern.h>
  72 #include <vm/vm_map.h>
  73 #include <vm/vm_extern.h>
  74
  75 #include <compat/ndis/pe_var.h>
  76 #include <compat/ndis/cfg_var.h>
  77 #include <compat/ndis/resource_var.h>
  78 #include <compat/ndis/ntoskrnl_var.h>
  79 #include <compat/ndis/hal_var.h>
  80 #include <compat/ndis/ndis_var.h>
  81
  82 #ifdef NTOSKRNL_DEBUG_TIMERS
  83 static int sysctl_show_timers(SYSCTL_HANDLER_ARGS);
  84
  85 SYSCTL_PROC(_debug, OID_AUTO, ntoskrnl_timers,
  86     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0,
  87     sysctl_show_timers, "I",
  88     "Show ntoskrnl timer stats");
  89 #endif
  90
  91 struct kdpc_queue {
  92         list_entry              kq_disp;
  93         struct thread           *kq_td;
  94         int                     kq_cpu;
  95         int                     kq_exit;
  96         int                     kq_running;
  97         kspin_lock              kq_lock;
  98         nt_kevent               kq_proc;
  99         nt_kevent               kq_done;
 100 };
 101
 102 typedef struct kdpc_queue kdpc_queue;
 103
 104 struct wb_ext {
 105         struct cv               we_cv;
 106         struct thread           *we_td;
 107 };
 108
 109 typedef struct wb_ext wb_ext;
 110
 111 #define NTOSKRNL_TIMEOUTS       256
 112 #ifdef NTOSKRNL_DEBUG_TIMERS
 113 static uint64_t ntoskrnl_timer_fires;
 114 static uint64_t ntoskrnl_timer_sets;
 115 static uint64_t ntoskrnl_timer_reloads;
 116 static uint64_t ntoskrnl_timer_cancels;
 117 #endif
 118
 119 struct callout_entry {
 120         struct callout          ce_callout;
 121         list_entry              ce_list;
 122 };
 123
 124 typedef struct callout_entry callout_entry;
 125
 126 static struct list_entry ntoskrnl_calllist;
 127 static struct mtx ntoskrnl_calllock;
 128 struct kuser_shared_data kuser_shared_data;
 129
 130 static struct list_entry ntoskrnl_intlist;
 131 static kspin_lock ntoskrnl_intlock;
 132
 133 static uint8_t RtlEqualUnicodeString(unicode_string *,
 134         unicode_string *, uint8_t);
 135 static void RtlCopyString(ansi_string *, const ansi_string *);
 136 static void RtlCopyUnicodeString(unicode_string *,
 137         unicode_string *);
 138 static irp *IoBuildSynchronousFsdRequest(uint32_t, device_object *,
 139          void *, uint32_t, uint64_t *, nt_kevent *, io_status_block *);
 140 static irp *IoBuildAsynchronousFsdRequest(uint32_t,
 141         device_object *, void *, uint32_t, uint64_t *, io_status_block *);
 142 static irp *IoBuildDeviceIoControlRequest(uint32_t,
 143         device_object *, void *, uint32_t, void *, uint32_t,
 144         uint8_t, nt_kevent *, io_status_block *);
 145 static irp *IoAllocateIrp(uint8_t, uint8_t);
 146 static void IoReuseIrp(irp *, uint32_t);
 147 static void IoFreeIrp(irp *);
 148 static void IoInitializeIrp(irp *, uint16_t, uint8_t);
 149 static irp *IoMakeAssociatedIrp(irp *, uint8_t);
 150 static uint32_t KeWaitForMultipleObjects(uint32_t,
 151         nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t,
 152         int64_t *, wait_block *);
 153 static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t);
 154 static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *);
 155 static void ntoskrnl_satisfy_multiple_waits(wait_block *);
 156 static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *);
 157 static void ntoskrnl_insert_timer(ktimer *, int);
 158 static void ntoskrnl_remove_timer(ktimer *);
 159 #ifdef NTOSKRNL_DEBUG_TIMERS
 160 static void ntoskrnl_show_timers(void);
 161 #endif
 162 static void ntoskrnl_timercall(void *);
 163 static void ntoskrnl_dpc_thread(void *);
 164 static void ntoskrnl_destroy_dpc_threads(void);
 165 static void ntoskrnl_destroy_workitem_threads(void);
 166 static void ntoskrnl_workitem_thread(void *);
 167 static void ntoskrnl_workitem(device_object *, void *);
 168 static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int);
 169 static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int);
 170 static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *);
 171 static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t);
 172 static uint16_t READ_REGISTER_USHORT(uint16_t *);
 173 static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t);
 174 static uint32_t READ_REGISTER_ULONG(uint32_t *);
 175 static void WRITE_REGISTER_UCHAR(uint8_t *, uint8_t);
 176 static uint8_t READ_REGISTER_UCHAR(uint8_t *);
 177 static int64_t _allmul(int64_t, int64_t);
 178 static int64_t _alldiv(int64_t, int64_t);
 179 static int64_t _allrem(int64_t, int64_t);
 180 static int64_t _allshr(int64_t, uint8_t);
 181 static int64_t _allshl(int64_t, uint8_t);
 182 static uint64_t _aullmul(uint64_t, uint64_t);
 183 static uint64_t _aulldiv(uint64_t, uint64_t);
 184 static uint64_t _aullrem(uint64_t, uint64_t);
 185 static uint64_t _aullshr(uint64_t, uint8_t);
 186 static uint64_t _aullshl(uint64_t, uint8_t);
 187 static slist_entry *ntoskrnl_pushsl(slist_header *, slist_entry *);
 188 static void InitializeSListHead(slist_header *);
 189 static slist_entry *ntoskrnl_popsl(slist_header *);
 190 static void ExFreePoolWithTag(void *, uint32_t);
 191 static void ExInitializePagedLookasideList(paged_lookaside_list *,
 192         lookaside_alloc_func *, lookaside_free_func *,
 193         uint32_t, size_t, uint32_t, uint16_t);
 194 static void ExDeletePagedLookasideList(paged_lookaside_list *);
 195 static void ExInitializeNPagedLookasideList(npaged_lookaside_list *,
 196         lookaside_alloc_func *, lookaside_free_func *,
 197         uint32_t, size_t, uint32_t, uint16_t);
 198 static void ExDeleteNPagedLookasideList(npaged_lookaside_list *);
 199 static slist_entry
 200         *ExInterlockedPushEntrySList(slist_header *,
 201         slist_entry *, kspin_lock *);
 202 static slist_entry
 203         *ExInterlockedPopEntrySList(slist_header *, kspin_lock *);
 204 static uint32_t InterlockedIncrement(volatile uint32_t *);
 205 static uint32_t InterlockedDecrement(volatile uint32_t *);
 206 static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t);
 207 static void *MmAllocateContiguousMemory(uint32_t, uint64_t);
 208 static void *MmAllocateContiguousMemorySpecifyCache(uint32_t,
 209         uint64_t, uint64_t, uint64_t, enum nt_caching_type);
 210 static void MmFreeContiguousMemory(void *);
 211 static void MmFreeContiguousMemorySpecifyCache(void *, uint32_t,
 212         enum nt_caching_type);
 213 static uint32_t MmSizeOfMdl(void *, size_t);
 214 static void *MmMapLockedPages(mdl *, uint8_t);
 215 static void *MmMapLockedPagesSpecifyCache(mdl *,
 216         uint8_t, uint32_t, void *, uint32_t, uint32_t);
 217 static void MmUnmapLockedPages(void *, mdl *);
 218 static device_t ntoskrnl_finddev(device_t, uint64_t, struct resource **);
 219 static void RtlZeroMemory(void *, size_t);
 220 static void RtlSecureZeroMemory(void *, size_t);
 221 static void RtlFillMemory(void *, size_t, uint8_t);
 222 static void RtlMoveMemory(void *, const void *, size_t);
 223 static ndis_status RtlCharToInteger(const char *, uint32_t, uint32_t *);
 224 static void RtlCopyMemory(void *, const void *, size_t);
 225 static size_t RtlCompareMemory(const void *, const void *, size_t);
 226 static ndis_status RtlUnicodeStringToInteger(unicode_string *,
 227         uint32_t, uint32_t *);
 228 static int atoi (const char *);
 229 static long atol (const char *);
 230 static int rand(void);
 231 static void srand(unsigned int);
 232 static void KeQuerySystemTime(uint64_t *);
 233 static uint32_t KeTickCount(void);
 234 static uint8_t IoIsWdmVersionAvailable(uint8_t, uint8_t);
 235 static int32_t IoOpenDeviceRegistryKey(struct device_object *, uint32_t,
 236     uint32_t, void **);
 237 static void ntoskrnl_thrfunc(void *);
 238 static ndis_status PsCreateSystemThread(ndis_handle *,
 239         uint32_t, void *, ndis_handle, void *, void *, void *);
 240 static ndis_status PsTerminateSystemThread(ndis_status);
 241 static ndis_status IoGetDeviceObjectPointer(unicode_string *,
 242         uint32_t, void *, device_object *);
 243 static ndis_status IoGetDeviceProperty(device_object *, uint32_t,
 244         uint32_t, void *, uint32_t *);
 245 static void KeInitializeMutex(kmutant *, uint32_t);
 246 static uint32_t KeReleaseMutex(kmutant *, uint8_t);
 247 static uint32_t KeReadStateMutex(kmutant *);
 248 static ndis_status ObReferenceObjectByHandle(ndis_handle,
 249         uint32_t, void *, uint8_t, void **, void **);
 250 static void ObfDereferenceObject(void *);
 251 static uint32_t ZwClose(ndis_handle);
 252 static uint32_t WmiQueryTraceInformation(uint32_t, void *, uint32_t,
 253         uint32_t, void *);
 254 static uint32_t WmiTraceMessage(uint64_t, uint32_t, void *, uint16_t, ...);
 255 static uint32_t IoWMIRegistrationControl(device_object *, uint32_t);
 256 static void *ntoskrnl_memset(void *, int, size_t);
 257 static void *ntoskrnl_memmove(void *, void *, size_t);
 258 static void *ntoskrnl_memchr(void *, unsigned char, size_t);
 259 static char *ntoskrnl_strstr(char *, char *);
 260 static char *ntoskrnl_strncat(char *, char *, size_t);
 261 static int ntoskrnl_toupper(int);
 262 static int ntoskrnl_tolower(int);
 263 static funcptr ntoskrnl_findwrap(funcptr);
 264 static uint32_t DbgPrint(char *, ...);
 265 static void DbgBreakPoint(void);
 266 static void KeBugCheckEx(uint32_t, u_long, u_long, u_long, u_long);
 267 static int32_t KeDelayExecutionThread(uint8_t, uint8_t, int64_t *);
 268 static int32_t KeSetPriorityThread(struct thread *, int32_t);
 269 static void dummy(void);
 270
 271 static struct mtx ntoskrnl_dispatchlock;
 272 static struct mtx ntoskrnl_interlock;
 273 static kspin_lock ntoskrnl_cancellock;
 274 static int ntoskrnl_kth = 0;
 275 static struct nt_objref_head ntoskrnl_reflist;
 276 static uma_zone_t mdl_zone;
 277 static uma_zone_t iw_zone;
 278 static struct kdpc_queue *kq_queues;
 279 static struct kdpc_queue *wq_queues;
 280 static int wq_idx = 0;
 281
 282 int
 283 ntoskrnl_libinit()
 284 {
 285         image_patch_table       *patch;
 286         int                     error;
 287         struct proc             *p;
 288         kdpc_queue              *kq;
 289         callout_entry           *e;
 290         int                     i;
 291
 292         mtx_init(&ntoskrnl_dispatchlock,
 293             "ntoskrnl dispatch lock", MTX_NDIS_LOCK, MTX_DEF|MTX_RECURSE);
 294         mtx_init(&ntoskrnl_interlock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
 295         KeInitializeSpinLock(&ntoskrnl_cancellock);
 296         KeInitializeSpinLock(&ntoskrnl_intlock);
 297         TAILQ_INIT(&ntoskrnl_reflist);
 298
 299         InitializeListHead(&ntoskrnl_calllist);
 300         InitializeListHead(&ntoskrnl_intlist);
 301         mtx_init(&ntoskrnl_calllock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
 302
 303         kq_queues = ExAllocatePoolWithTag(NonPagedPool,
 304 #ifdef NTOSKRNL_MULTIPLE_DPCS
 305             sizeof(kdpc_queue) * mp_ncpus, 0);
 306 #else
 307             sizeof(kdpc_queue), 0);
 308 #endif
 309
 310         if (kq_queues == NULL)
 311                 return (ENOMEM);
 312
 313         wq_queues = ExAllocatePoolWithTag(NonPagedPool,
 314             sizeof(kdpc_queue) * WORKITEM_THREADS, 0);
 315
 316         if (wq_queues == NULL)
 317                 return (ENOMEM);
 318
 319 #ifdef NTOSKRNL_MULTIPLE_DPCS
 320         bzero((char *)kq_queues, sizeof(kdpc_queue) * mp_ncpus);
 321 #else
 322         bzero((char *)kq_queues, sizeof(kdpc_queue));
 323 #endif
 324         bzero((char *)wq_queues, sizeof(kdpc_queue) * WORKITEM_THREADS);
 325
 326         /*
 327          * Launch the DPC threads.
 328          */
 329
 330 #ifdef NTOSKRNL_MULTIPLE_DPCS
 331         for (i = 0; i < mp_ncpus; i++) {
 332 #else
 333         for (i = 0; i < 1; i++) {
 334 #endif
 335                 kq = kq_queues + i;
 336                 kq->kq_cpu = i;
 337                 error = kproc_create(ntoskrnl_dpc_thread, kq, &p,
 338                     RFHIGHPID, NDIS_KSTACK_PAGES, "Windows DPC %d", i);
 339                 if (error)
 340                         panic("failed to launch DPC thread");
 341         }
 342
 343         /*
 344          * Launch the workitem threads.
 345          */
 346
 347         for (i = 0; i < WORKITEM_THREADS; i++) {
 348                 kq = wq_queues + i;
 349                 error = kproc_create(ntoskrnl_workitem_thread, kq, &p,
 350                     RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Workitem %d", i);
 351                 if (error)
 352                         panic("failed to launch workitem thread");
 353         }
 354
 355         patch = ntoskrnl_functbl;
 356         while (patch->ipt_func != NULL) {
 357                 windrv_wrap((funcptr)patch->ipt_func,
 358                     (funcptr *)&patch->ipt_wrap,
 359                     patch->ipt_argcnt, patch->ipt_ftype);
 360                 patch++;
 361         }
 362
 363         for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) {
 364                 e = ExAllocatePoolWithTag(NonPagedPool,
 365                     sizeof(callout_entry), 0);
 366                 if (e == NULL)
 367                         panic("failed to allocate timeouts");
 368                 mtx_lock_spin(&ntoskrnl_calllock);
 369                 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
 370                 mtx_unlock_spin(&ntoskrnl_calllock);
 371         }
 372
 373         /*
 374          * MDLs are supposed to be variable size (they describe
 375          * buffers containing some number of pages, but we don't
 376          * know ahead of time how many pages that will be). But
 377          * always allocating them off the heap is very slow. As
 378          * a compromise, we create an MDL UMA zone big enough to
 379          * handle any buffer requiring up to 16 pages, and we
 380          * use those for any MDLs for buffers of 16 pages or less
 381          * in size. For buffers larger than that (which we assume
 382          * will be few and far between, we allocate the MDLs off
 383          * the heap.
 384          */
 385
 386         mdl_zone = uma_zcreate("Windows MDL", MDL_ZONE_SIZE,
 387             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 388
 389         iw_zone = uma_zcreate("Windows WorkItem", sizeof(io_workitem),
 390             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 391
 392         return (0);
 393 }
 394
 395 int
 396 ntoskrnl_libfini()
 397 {
 398         image_patch_table       *patch;
 399         callout_entry           *e;
 400         list_entry              *l;
 401
 402         patch = ntoskrnl_functbl;
 403         while (patch->ipt_func != NULL) {
 404                 windrv_unwrap(patch->ipt_wrap);
 405                 patch++;
 406         }
 407
 408         /* Stop the workitem queues. */
 409         ntoskrnl_destroy_workitem_threads();
 410         /* Stop the DPC queues. */
 411         ntoskrnl_destroy_dpc_threads();
 412
 413         ExFreePool(kq_queues);
 414         ExFreePool(wq_queues);
 415
 416         uma_zdestroy(mdl_zone);
 417         uma_zdestroy(iw_zone);
 418
 419         mtx_lock_spin(&ntoskrnl_calllock);
 420         while(!IsListEmpty(&ntoskrnl_calllist)) {
 421                 l = RemoveHeadList(&ntoskrnl_calllist);
 422                 e = CONTAINING_RECORD(l, callout_entry, ce_list);
 423                 mtx_unlock_spin(&ntoskrnl_calllock);
 424                 ExFreePool(e);
 425                 mtx_lock_spin(&ntoskrnl_calllock);
 426         }
 427         mtx_unlock_spin(&ntoskrnl_calllock);
 428
 429         mtx_destroy(&ntoskrnl_dispatchlock);
 430         mtx_destroy(&ntoskrnl_interlock);
 431         mtx_destroy(&ntoskrnl_calllock);
 432
 433         return (0);
 434 }
 435
 436 /*
 437  * We need to be able to reference this externally from the wrapper;
 438  * GCC only generates a local implementation of memset.
 439  */
 440 static void *
 441 ntoskrnl_memset(buf, ch, size)
 442         void                    *buf;
 443         int                     ch;
 444         size_t                  size;
 445 {
 446         return (memset(buf, ch, size));
 447 }
 448
 449 static void *
 450 ntoskrnl_memmove(dst, src, size)
 451         void                    *src;
 452         void                    *dst;
 453         size_t                  size;
 454 {
 455         bcopy(src, dst, size);
 456         return (dst);
 457 }
 458
 459 static void *
 460 ntoskrnl_memchr(void *buf, unsigned char ch, size_t len)
 461 {
 462         if (len != 0) {
 463                 unsigned char *p = buf;
 464
 465                 do {
 466                         if (*p++ == ch)
 467                                 return (p - 1);
 468                 } while (--len != 0);
 469         }
 470         return (NULL);
 471 }
 472
 473 static char *
 474 ntoskrnl_strstr(s, find)
 475         char *s, *find;
 476 {
 477         char c, sc;
 478         size_t len;
 479
 480         if ((c = *find++) != 0) {
 481                 len = strlen(find);
 482                 do {
 483                         do {
 484                                 if ((sc = *s++) == 0)
 485                                         return (NULL);
 486                         } while (sc != c);
 487                 } while (strncmp(s, find, len) != 0);
 488                 s--;
 489         }
 490         return ((char *)s);
 491 }
 492
 493 /* Taken from libc */
 494 static char *
 495 ntoskrnl_strncat(dst, src, n)
 496         char            *dst;
 497         char            *src;
 498         size_t          n;
 499 {
 500         if (n != 0) {
 501                 char *d = dst;
 502                 const char *s = src;
 503
 504                 while (*d != 0)
 505                         d++;
 506                 do {
 507                         if ((*d = *s++) == 0)
 508                                 break;
 509                         d++;
 510                 } while (--n != 0);
 511                 *d = 0;
 512         }
 513         return (dst);
 514 }
 515
 516 static int
 517 ntoskrnl_toupper(c)
 518         int                     c;
 519 {
 520         return (toupper(c));
 521 }
 522
 523 static int
 524 ntoskrnl_tolower(c)
 525         int                     c;
 526 {
 527         return (tolower(c));
 528 }
 529
 530 static uint8_t
 531 RtlEqualUnicodeString(unicode_string *str1, unicode_string *str2,
 532         uint8_t caseinsensitive)
 533 {
 534         int                     i;
 535
 536         if (str1->us_len != str2->us_len)
 537                 return (FALSE);
 538
 539         for (i = 0; i < str1->us_len; i++) {
 540                 if (caseinsensitive == TRUE) {
 541                         if (toupper((char)(str1->us_buf[i] & 0xFF)) !=
 542                             toupper((char)(str2->us_buf[i] & 0xFF)))
 543                                 return (FALSE);
 544                 } else {
 545                         if (str1->us_buf[i] != str2->us_buf[i])
 546                                 return (FALSE);
 547                 }
 548         }
 549
 550         return (TRUE);
 551 }
 552
 553 static void
 554 RtlCopyString(dst, src)
 555         ansi_string             *dst;
 556         const ansi_string       *src;
 557 {
 558         if (src != NULL && src->as_buf != NULL && dst->as_buf != NULL) {
 559                 dst->as_len = min(src->as_len, dst->as_maxlen);
 560                 memcpy(dst->as_buf, src->as_buf, dst->as_len);
 561                 if (dst->as_len < dst->as_maxlen)
 562                         dst->as_buf[dst->as_len] = 0;
 563         } else
 564                 dst->as_len = 0;
 565 }
 566
 567 static void
 568 RtlCopyUnicodeString(dest, src)
 569         unicode_string          *dest;
 570         unicode_string          *src;
 571 {
 572
 573         if (dest->us_maxlen >= src->us_len)
 574                 dest->us_len = src->us_len;
 575         else
 576                 dest->us_len = dest->us_maxlen;
 577         memcpy(dest->us_buf, src->us_buf, dest->us_len);
 578 }
 579
 580 static void
 581 ntoskrnl_ascii_to_unicode(ascii, unicode, len)
 582         char                    *ascii;
 583         uint16_t                *unicode;
 584         int                     len;
 585 {
 586         int                     i;
 587         uint16_t                *ustr;
 588
 589         ustr = unicode;
 590         for (i = 0; i < len; i++) {
 591                 *ustr = (uint16_t)ascii[i];
 592                 ustr++;
 593         }
 594 }
 595
 596 static void
 597 ntoskrnl_unicode_to_ascii(unicode, ascii, len)
 598         uint16_t                *unicode;
 599         char                    *ascii;
 600         int                     len;
 601 {
 602         int                     i;
 603         uint8_t                 *astr;
 604
 605         astr = ascii;
 606         for (i = 0; i < len / 2; i++) {
 607                 *astr = (uint8_t)unicode[i];
 608                 astr++;
 609         }
 610 }
 611
 612 uint32_t
 613 RtlUnicodeStringToAnsiString(ansi_string *dest, unicode_string *src, uint8_t allocate)
 614 {
 615         if (dest == NULL || src == NULL)
 616                 return (STATUS_INVALID_PARAMETER);
 617
 618         dest->as_len = src->us_len / 2;
 619         if (dest->as_maxlen < dest->as_len)
 620                 dest->as_len = dest->as_maxlen;
 621
 622         if (allocate == TRUE) {
 623                 dest->as_buf = ExAllocatePoolWithTag(NonPagedPool,
 624                     (src->us_len / 2) + 1, 0);
 625                 if (dest->as_buf == NULL)
 626                         return (STATUS_INSUFFICIENT_RESOURCES);
 627                 dest->as_len = dest->as_maxlen = src->us_len / 2;
 628         } else {
 629                 dest->as_len = src->us_len / 2; /* XXX */
 630                 if (dest->as_maxlen < dest->as_len)
 631                         dest->as_len = dest->as_maxlen;
 632         }
 633
 634         ntoskrnl_unicode_to_ascii(src->us_buf, dest->as_buf,
 635             dest->as_len * 2);
 636
 637         return (STATUS_SUCCESS);
 638 }
 639
 640 uint32_t
 641 RtlAnsiStringToUnicodeString(unicode_string *dest, ansi_string *src,
 642         uint8_t allocate)
 643 {
 644         if (dest == NULL || src == NULL)
 645                 return (STATUS_INVALID_PARAMETER);
 646
 647         if (allocate == TRUE) {
 648                 dest->us_buf = ExAllocatePoolWithTag(NonPagedPool,
 649                     src->as_len * 2, 0);
 650                 if (dest->us_buf == NULL)
 651                         return (STATUS_INSUFFICIENT_RESOURCES);
 652                 dest->us_len = dest->us_maxlen = strlen(src->as_buf) * 2;
 653         } else {
 654                 dest->us_len = src->as_len * 2; /* XXX */
 655                 if (dest->us_maxlen < dest->us_len)
 656                         dest->us_len = dest->us_maxlen;
 657         }
 658
 659         ntoskrnl_ascii_to_unicode(src->as_buf, dest->us_buf,
 660             dest->us_len / 2);
 661
 662         return (STATUS_SUCCESS);
 663 }
 664
 665 void *
 666 ExAllocatePoolWithTag(pooltype, len, tag)
 667         uint32_t                pooltype;
 668         size_t                  len;
 669         uint32_t                tag;
 670 {
 671         void                    *buf;
 672
 673         buf = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO);
 674         if (buf == NULL)
 675                 return (NULL);
 676
 677         return (buf);
 678 }
 679
 680 static void
 681 ExFreePoolWithTag(buf, tag)
 682         void            *buf;
 683         uint32_t        tag;
 684 {
 685         ExFreePool(buf);
 686 }
 687
 688 void
 689 ExFreePool(buf)
 690         void                    *buf;
 691 {
 692         free(buf, M_DEVBUF);
 693 }
 694
 695 uint32_t
 696 IoAllocateDriverObjectExtension(drv, clid, extlen, ext)
 697         driver_object           *drv;
 698         void                    *clid;
 699         uint32_t                extlen;
 700         void                    **ext;
 701 {
 702         custom_extension        *ce;
 703
 704         ce = ExAllocatePoolWithTag(NonPagedPool, sizeof(custom_extension)
 705             + extlen, 0);
 706
 707         if (ce == NULL)
 708                 return (STATUS_INSUFFICIENT_RESOURCES);
 709
 710         ce->ce_clid = clid;
 711         InsertTailList((&drv->dro_driverext->dre_usrext), (&ce->ce_list));
 712
 713         *ext = (void *)(ce + 1);
 714
 715         return (STATUS_SUCCESS);
 716 }
 717
 718 void *
 719 IoGetDriverObjectExtension(drv, clid)
 720         driver_object           *drv;
 721         void                    *clid;
 722 {
 723         list_entry              *e;
 724         custom_extension        *ce;
 725
 726         /*
 727          * Sanity check. Our dummy bus drivers don't have
 728          * any driver extensions.
 729          */
 730
 731         if (drv->dro_driverext == NULL)
 732                 return (NULL);
 733
 734         e = drv->dro_driverext->dre_usrext.nle_flink;
 735         while (e != &drv->dro_driverext->dre_usrext) {
 736                 ce = (custom_extension *)e;
 737                 if (ce->ce_clid == clid)
 738                         return ((void *)(ce + 1));
 739                 e = e->nle_flink;
 740         }
 741
 742         return (NULL);
 743 }
 744
 745 uint32_t
 746 IoCreateDevice(driver_object *drv, uint32_t devextlen, unicode_string *devname,
 747         uint32_t devtype, uint32_t devchars, uint8_t exclusive,
 748         device_object **newdev)
 749 {
 750         device_object           *dev;
 751
 752         dev = ExAllocatePoolWithTag(NonPagedPool, sizeof(device_object), 0);
 753         if (dev == NULL)
 754                 return (STATUS_INSUFFICIENT_RESOURCES);
 755
 756         dev->do_type = devtype;
 757         dev->do_drvobj = drv;
 758         dev->do_currirp = NULL;
 759         dev->do_flags = 0;
 760
 761         if (devextlen) {
 762                 dev->do_devext = ExAllocatePoolWithTag(NonPagedPool,
 763                     devextlen, 0);
 764
 765                 if (dev->do_devext == NULL) {
 766                         ExFreePool(dev);
 767                         return (STATUS_INSUFFICIENT_RESOURCES);
 768                 }
 769
 770                 bzero(dev->do_devext, devextlen);
 771         } else
 772                 dev->do_devext = NULL;
 773
 774         dev->do_size = sizeof(device_object) + devextlen;
 775         dev->do_refcnt = 1;
 776         dev->do_attacheddev = NULL;
 777         dev->do_nextdev = NULL;
 778         dev->do_devtype = devtype;
 779         dev->do_stacksize = 1;
 780         dev->do_alignreq = 1;
 781         dev->do_characteristics = devchars;
 782         dev->do_iotimer = NULL;
 783         KeInitializeEvent(&dev->do_devlock, EVENT_TYPE_SYNC, TRUE);
 784
 785         /*
 786          * Vpd is used for disk/tape devices,
 787          * but we don't support those. (Yet.)
 788          */
 789         dev->do_vpb = NULL;
 790
 791         dev->do_devobj_ext = ExAllocatePoolWithTag(NonPagedPool,
 792             sizeof(devobj_extension), 0);
 793
 794         if (dev->do_devobj_ext == NULL) {
 795                 if (dev->do_devext != NULL)
 796                         ExFreePool(dev->do_devext);
 797                 ExFreePool(dev);
 798                 return (STATUS_INSUFFICIENT_RESOURCES);
 799         }
 800
 801         dev->do_devobj_ext->dve_type = 0;
 802         dev->do_devobj_ext->dve_size = sizeof(devobj_extension);
 803         dev->do_devobj_ext->dve_devobj = dev;
 804
 805         /*
 806          * Attach this device to the driver object's list
 807          * of devices. Note: this is not the same as attaching
 808          * the device to the device stack. The driver's AddDevice
 809          * routine must explicitly call IoAddDeviceToDeviceStack()
 810          * to do that.
 811          */
 812
 813         if (drv->dro_devobj == NULL) {
 814                 drv->dro_devobj = dev;
 815                 dev->do_nextdev = NULL;
 816         } else {
 817                 dev->do_nextdev = drv->dro_devobj;
 818                 drv->dro_devobj = dev;
 819         }
 820
 821         *newdev = dev;
 822
 823         return (STATUS_SUCCESS);
 824 }
 825
 826 void
 827 IoDeleteDevice(dev)
 828         device_object           *dev;
 829 {
 830         device_object           *prev;
 831
 832         if (dev == NULL)
 833                 return;
 834
 835         if (dev->do_devobj_ext != NULL)
 836                 ExFreePool(dev->do_devobj_ext);
 837
 838         if (dev->do_devext != NULL)
 839                 ExFreePool(dev->do_devext);
 840
 841         /* Unlink the device from the driver's device list. */
 842
 843         prev = dev->do_drvobj->dro_devobj;
 844         if (prev == dev)
 845                 dev->do_drvobj->dro_devobj = dev->do_nextdev;
 846         else {
 847                 while (prev->do_nextdev != dev)
 848                         prev = prev->do_nextdev;
 849                 prev->do_nextdev = dev->do_nextdev;
 850         }
 851
 852         ExFreePool(dev);
 853 }
 854
 855 device_object *
 856 IoGetAttachedDevice(dev)
 857         device_object           *dev;
 858 {
 859         device_object           *d;
 860
 861         if (dev == NULL)
 862                 return (NULL);
 863
 864         d = dev;
 865
 866         while (d->do_attacheddev != NULL)
 867                 d = d->do_attacheddev;
 868
 869         return (d);
 870 }
 871
 872 static irp *
 873 IoBuildSynchronousFsdRequest(func, dobj, buf, len, off, event, status)
 874         uint32_t                func;
 875         device_object           *dobj;
 876         void                    *buf;
 877         uint32_t                len;
 878         uint64_t                *off;
 879         nt_kevent               *event;
 880         io_status_block         *status;
 881 {
 882         irp                     *ip;
 883
 884         ip = IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status);
 885         if (ip == NULL)
 886                 return (NULL);
 887         ip->irp_usrevent = event;
 888
 889         return (ip);
 890 }
 891
 892 static irp *
 893 IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status)
 894         uint32_t                func;
 895         device_object           *dobj;
 896         void                    *buf;
 897         uint32_t                len;
 898         uint64_t                *off;
 899         io_status_block         *status;
 900 {
 901         irp                     *ip;
 902         io_stack_location       *sl;
 903
 904         ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
 905         if (ip == NULL)
 906                 return (NULL);
 907
 908         ip->irp_usriostat = status;
 909         ip->irp_tail.irp_overlay.irp_thread = NULL;
 910
 911         sl = IoGetNextIrpStackLocation(ip);
 912         sl->isl_major = func;
 913         sl->isl_minor = 0;
 914         sl->isl_flags = 0;
 915         sl->isl_ctl = 0;
 916         sl->isl_devobj = dobj;
 917         sl->isl_fileobj = NULL;
 918         sl->isl_completionfunc = NULL;
 919
 920         ip->irp_userbuf = buf;
 921
 922         if (dobj->do_flags & DO_BUFFERED_IO) {
 923                 ip->irp_assoc.irp_sysbuf =
 924                     ExAllocatePoolWithTag(NonPagedPool, len, 0);
 925                 if (ip->irp_assoc.irp_sysbuf == NULL) {
 926                         IoFreeIrp(ip);
 927                         return (NULL);
 928                 }
 929                 bcopy(buf, ip->irp_assoc.irp_sysbuf, len);
 930         }
 931
 932         if (dobj->do_flags & DO_DIRECT_IO) {
 933                 ip->irp_mdl = IoAllocateMdl(buf, len, FALSE, FALSE, ip);
 934                 if (ip->irp_mdl == NULL) {
 935                         if (ip->irp_assoc.irp_sysbuf != NULL)
 936                                 ExFreePool(ip->irp_assoc.irp_sysbuf);
 937                         IoFreeIrp(ip);
 938                         return (NULL);
 939                 }
 940                 ip->irp_userbuf = NULL;
 941                 ip->irp_assoc.irp_sysbuf = NULL;
 942         }
 943
 944         if (func == IRP_MJ_READ) {
 945                 sl->isl_parameters.isl_read.isl_len = len;
 946                 if (off != NULL)
 947                         sl->isl_parameters.isl_read.isl_byteoff = *off;
 948                 else
 949                         sl->isl_parameters.isl_read.isl_byteoff = 0;
 950         }
 951
 952         if (func == IRP_MJ_WRITE) {
 953                 sl->isl_parameters.isl_write.isl_len = len;
 954                 if (off != NULL)
 955                         sl->isl_parameters.isl_write.isl_byteoff = *off;
 956                 else
 957                         sl->isl_parameters.isl_write.isl_byteoff = 0;
 958         }
 959
 960         return (ip);
 961 }
 962
 963 static irp *
 964 IoBuildDeviceIoControlRequest(uint32_t iocode, device_object *dobj, void *ibuf,
 965         uint32_t ilen, void *obuf, uint32_t olen, uint8_t isinternal,
 966         nt_kevent *event, io_status_block *status)
 967 {
 968         irp                     *ip;
 969         io_stack_location       *sl;
 970         uint32_t                buflen;
 971
 972         ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
 973         if (ip == NULL)
 974                 return (NULL);
 975         ip->irp_usrevent = event;
 976         ip->irp_usriostat = status;
 977         ip->irp_tail.irp_overlay.irp_thread = NULL;
 978
 979         sl = IoGetNextIrpStackLocation(ip);
 980         sl->isl_major = isinternal == TRUE ?
 981             IRP_MJ_INTERNAL_DEVICE_CONTROL : IRP_MJ_DEVICE_CONTROL;
 982         sl->isl_minor = 0;
 983         sl->isl_flags = 0;
 984         sl->isl_ctl = 0;
 985         sl->isl_devobj = dobj;
 986         sl->isl_fileobj = NULL;
 987         sl->isl_completionfunc = NULL;
 988         sl->isl_parameters.isl_ioctl.isl_iocode = iocode;
 989         sl->isl_parameters.isl_ioctl.isl_ibuflen = ilen;
 990         sl->isl_parameters.isl_ioctl.isl_obuflen = olen;
 991
 992         switch(IO_METHOD(iocode)) {
 993         case METHOD_BUFFERED:
 994                 if (ilen > olen)
 995                         buflen = ilen;
 996                 else
 997                         buflen = olen;
 998                 if (buflen) {
 999                         ip->irp_assoc.irp_sysbuf =
1000                             ExAllocatePoolWithTag(NonPagedPool, buflen, 0);
1001                         if (ip->irp_assoc.irp_sysbuf == NULL) {
1002                                 IoFreeIrp(ip);
1003                                 return (NULL);
1004                         }
1005                 }
1006                 if (ilen && ibuf != NULL) {
1007                         bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1008                         bzero((char *)ip->irp_assoc.irp_sysbuf + ilen,
1009                             buflen - ilen);
1010                 } else
1011                         bzero(ip->irp_assoc.irp_sysbuf, ilen);
1012                 ip->irp_userbuf = obuf;
1013                 break;
1014         case METHOD_IN_DIRECT:
1015         case METHOD_OUT_DIRECT:
1016                 if (ilen && ibuf != NULL) {
1017                         ip->irp_assoc.irp_sysbuf =
1018                             ExAllocatePoolWithTag(NonPagedPool, ilen, 0);
1019                         if (ip->irp_assoc.irp_sysbuf == NULL) {
1020                                 IoFreeIrp(ip);
1021                                 return (NULL);
1022                         }
1023                         bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1024                 }
1025                 if (olen && obuf != NULL) {
1026                         ip->irp_mdl = IoAllocateMdl(obuf, olen,
1027                             FALSE, FALSE, ip);
1028                         /*
1029                          * Normally we would MmProbeAndLockPages()
1030                          * here, but we don't have to in our
1031                          * imlementation.
1032                          */
1033                 }
1034                 break;
1035         case METHOD_NEITHER:
1036                 ip->irp_userbuf = obuf;
1037                 sl->isl_parameters.isl_ioctl.isl_type3ibuf = ibuf;
1038                 break;
1039         default:
1040                 break;
1041         }
1042
1043         /*
1044          * Ideally, we should associate this IRP with the calling
1045          * thread here.
1046          */
1047
1048         return (ip);
1049 }
1050
1051 static irp *
1052 IoAllocateIrp(uint8_t stsize, uint8_t chargequota)
1053 {
1054         irp                     *i;
1055
1056         i = ExAllocatePoolWithTag(NonPagedPool, IoSizeOfIrp(stsize), 0);
1057         if (i == NULL)
1058                 return (NULL);
1059
1060         IoInitializeIrp(i, IoSizeOfIrp(stsize), stsize);
1061
1062         return (i);
1063 }
1064
1065 static irp *
1066 IoMakeAssociatedIrp(irp *ip, uint8_t stsize)
1067 {
1068         irp                     *associrp;
1069
1070         associrp = IoAllocateIrp(stsize, FALSE);
1071         if (associrp == NULL)
1072                 return (NULL);
1073
1074         mtx_lock(&ntoskrnl_dispatchlock);
1075         associrp->irp_flags |= IRP_ASSOCIATED_IRP;
1076         associrp->irp_tail.irp_overlay.irp_thread =
1077             ip->irp_tail.irp_overlay.irp_thread;
1078         associrp->irp_assoc.irp_master = ip;
1079         mtx_unlock(&ntoskrnl_dispatchlock);
1080
1081         return (associrp);
1082 }
1083
1084 static void
1085 IoFreeIrp(ip)
1086         irp                     *ip;
1087 {
1088         ExFreePool(ip);
1089 }
1090
1091 static void
1092 IoInitializeIrp(irp *io, uint16_t psize, uint8_t ssize)
1093 {
1094         bzero((char *)io, IoSizeOfIrp(ssize));
1095         io->irp_size = psize;
1096         io->irp_stackcnt = ssize;
1097         io->irp_currentstackloc = ssize;
1098         InitializeListHead(&io->irp_thlist);
1099         io->irp_tail.irp_overlay.irp_csl =
1100             (io_stack_location *)(io + 1) + ssize;
1101 }
1102
1103 static void
1104 IoReuseIrp(ip, status)
1105         irp                     *ip;
1106         uint32_t                status;
1107 {
1108         uint8_t                 allocflags;
1109
1110         allocflags = ip->irp_allocflags;
1111         IoInitializeIrp(ip, ip->irp_size, ip->irp_stackcnt);
1112         ip->irp_iostat.isb_status = status;
1113         ip->irp_allocflags = allocflags;
1114 }
1115
1116 void
1117 IoAcquireCancelSpinLock(uint8_t *irql)
1118 {
1119         KeAcquireSpinLock(&ntoskrnl_cancellock, irql);
1120 }
1121
1122 void
1123 IoReleaseCancelSpinLock(uint8_t irql)
1124 {
1125         KeReleaseSpinLock(&ntoskrnl_cancellock, irql);
1126 }
1127
1128 uint8_t
1129 IoCancelIrp(irp *ip)
1130 {
1131         cancel_func             cfunc;
1132         uint8_t                 cancelirql;
1133
1134         IoAcquireCancelSpinLock(&cancelirql);
1135         cfunc = IoSetCancelRoutine(ip, NULL);
1136         ip->irp_cancel = TRUE;
1137         if (cfunc == NULL) {
1138                 IoReleaseCancelSpinLock(cancelirql);
1139                 return (FALSE);
1140         }
1141         ip->irp_cancelirql = cancelirql;
1142         MSCALL2(cfunc, IoGetCurrentIrpStackLocation(ip)->isl_devobj, ip);
1143         return (uint8_t)IoSetCancelValue(ip, TRUE);
1144 }
1145
1146 uint32_t
1147 IofCallDriver(dobj, ip)
1148         device_object           *dobj;
1149         irp                     *ip;
1150 {
1151         driver_object           *drvobj;
1152         io_stack_location       *sl;
1153         uint32_t                status;
1154         driver_dispatch         disp;
1155
1156         drvobj = dobj->do_drvobj;
1157
1158         if (ip->irp_currentstackloc <= 0)
1159                 panic("IoCallDriver(): out of stack locations");
1160
1161         IoSetNextIrpStackLocation(ip);
1162         sl = IoGetCurrentIrpStackLocation(ip);
1163
1164         sl->isl_devobj = dobj;
1165
1166         disp = drvobj->dro_dispatch[sl->isl_major];
1167         status = MSCALL2(disp, dobj, ip);
1168
1169         return (status);
1170 }
1171
1172 void
1173 IofCompleteRequest(irp *ip, uint8_t prioboost)
1174 {
1175         uint32_t                status;
1176         device_object           *dobj;
1177         io_stack_location       *sl;
1178         completion_func         cf;
1179
1180         KASSERT(ip->irp_iostat.isb_status != STATUS_PENDING,
1181             ("incorrect IRP(%p) status (STATUS_PENDING)", ip));
1182
1183         sl = IoGetCurrentIrpStackLocation(ip);
1184         IoSkipCurrentIrpStackLocation(ip);
1185
1186         do {
1187                 if (sl->isl_ctl & SL_PENDING_RETURNED)
1188                         ip->irp_pendingreturned = TRUE;
1189
1190                 if (ip->irp_currentstackloc != (ip->irp_stackcnt + 1))
1191                         dobj = IoGetCurrentIrpStackLocation(ip)->isl_devobj;
1192                 else
1193                         dobj = NULL;
1194
1195                 if (sl->isl_completionfunc != NULL &&
1196                     ((ip->irp_iostat.isb_status == STATUS_SUCCESS &&
1197                     sl->isl_ctl & SL_INVOKE_ON_SUCCESS) ||
1198                     (ip->irp_iostat.isb_status != STATUS_SUCCESS &&
1199                     sl->isl_ctl & SL_INVOKE_ON_ERROR) ||
1200                     (ip->irp_cancel == TRUE &&
1201                     sl->isl_ctl & SL_INVOKE_ON_CANCEL))) {
1202                         cf = sl->isl_completionfunc;
1203                         status = MSCALL3(cf, dobj, ip, sl->isl_completionctx);
1204                         if (status == STATUS_MORE_PROCESSING_REQUIRED)
1205                                 return;
1206                 } else {
1207                         if ((ip->irp_currentstackloc <= ip->irp_stackcnt) &&
1208                             (ip->irp_pendingreturned == TRUE))
1209                                 IoMarkIrpPending(ip);
1210                 }
1211
1212                 /* move to the next.  */
1213                 IoSkipCurrentIrpStackLocation(ip);
1214                 sl++;
1215         } while (ip->irp_currentstackloc <= (ip->irp_stackcnt + 1));
1216
1217         if (ip->irp_usriostat != NULL)
1218                 *ip->irp_usriostat = ip->irp_iostat;
1219         if (ip->irp_usrevent != NULL)
1220                 KeSetEvent(ip->irp_usrevent, prioboost, FALSE);
1221
1222         /* Handle any associated IRPs. */
1223
1224         if (ip->irp_flags & IRP_ASSOCIATED_IRP) {
1225                 uint32_t                masterirpcnt;
1226                 irp                     *masterirp;
1227                 mdl                     *m;
1228
1229                 masterirp = ip->irp_assoc.irp_master;
1230                 masterirpcnt =
1231                     InterlockedDecrement(&masterirp->irp_assoc.irp_irpcnt);
1232
1233                 while ((m = ip->irp_mdl) != NULL) {
1234                         ip->irp_mdl = m->mdl_next;
1235                         IoFreeMdl(m);
1236                 }
1237                 IoFreeIrp(ip);
1238                 if (masterirpcnt == 0)
1239                         IoCompleteRequest(masterirp, IO_NO_INCREMENT);
1240                 return;
1241         }
1242
1243         /* With any luck, these conditions will never arise. */
1244
1245         if (ip->irp_flags & IRP_PAGING_IO) {
1246                 if (ip->irp_mdl != NULL)
1247                         IoFreeMdl(ip->irp_mdl);
1248                 IoFreeIrp(ip);
1249         }
1250 }
1251
1252 void
1253 ntoskrnl_intr(arg)
1254         void                    *arg;
1255 {
1256         kinterrupt              *iobj;
1257         uint8_t                 irql;
1258         uint8_t                 claimed;
1259         list_entry              *l;
1260
1261         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1262         l = ntoskrnl_intlist.nle_flink;
1263         while (l != &ntoskrnl_intlist) {
1264                 iobj = CONTAINING_RECORD(l, kinterrupt, ki_list);
1265                 claimed = MSCALL2(iobj->ki_svcfunc, iobj, iobj->ki_svcctx);
1266                 if (claimed == TRUE)
1267                         break;
1268                 l = l->nle_flink;
1269         }
1270         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1271 }
1272
1273 uint8_t
1274 KeAcquireInterruptSpinLock(iobj)
1275         kinterrupt              *iobj;
1276 {
1277         uint8_t                 irql;
1278         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1279         return (irql);
1280 }
1281
1282 void
1283 KeReleaseInterruptSpinLock(kinterrupt *iobj, uint8_t irql)
1284 {
1285         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1286 }
1287
1288 uint8_t
1289 KeSynchronizeExecution(iobj, syncfunc, syncctx)
1290         kinterrupt              *iobj;
1291         void                    *syncfunc;
1292         void                    *syncctx;
1293 {
1294         uint8_t                 irql;
1295
1296         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1297         MSCALL1(syncfunc, syncctx);
1298         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1299
1300         return (TRUE);
1301 }
1302
1303 /*
1304  * IoConnectInterrupt() is passed only the interrupt vector and
1305  * irql that a device wants to use, but no device-specific tag
1306  * of any kind. This conflicts rather badly with FreeBSD's
1307  * bus_setup_intr(), which needs the device_t for the device
1308  * requesting interrupt delivery. In order to bypass this
1309  * inconsistency, we implement a second level of interrupt
1310  * dispatching on top of bus_setup_intr(). All devices use
1311  * ntoskrnl_intr() as their ISR, and any device requesting
1312  * interrupts will be registered with ntoskrnl_intr()'s interrupt
1313  * dispatch list. When an interrupt arrives, we walk the list
1314  * and invoke all the registered ISRs. This effectively makes all
1315  * interrupts shared, but it's the only way to duplicate the
1316  * semantics of IoConnectInterrupt() and IoDisconnectInterrupt() properly.
1317  */
1318
1319 uint32_t
1320 IoConnectInterrupt(kinterrupt **iobj, void *svcfunc, void *svcctx,
1321         kspin_lock *lock, uint32_t vector, uint8_t irql, uint8_t syncirql,
1322         uint8_t imode, uint8_t shared, uint32_t affinity, uint8_t savefloat)
1323 {
1324         uint8_t                 curirql;
1325
1326         *iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0);
1327         if (*iobj == NULL)
1328                 return (STATUS_INSUFFICIENT_RESOURCES);
1329
1330         (*iobj)->ki_svcfunc = svcfunc;
1331         (*iobj)->ki_svcctx = svcctx;
1332
1333         if (lock == NULL) {
1334                 KeInitializeSpinLock(&(*iobj)->ki_lock_priv);
1335                 (*iobj)->ki_lock = &(*iobj)->ki_lock_priv;
1336         } else
1337                 (*iobj)->ki_lock = lock;
1338
1339         KeAcquireSpinLock(&ntoskrnl_intlock, &curirql);
1340         InsertHeadList((&ntoskrnl_intlist), (&(*iobj)->ki_list));
1341         KeReleaseSpinLock(&ntoskrnl_intlock, curirql);
1342
1343         return (STATUS_SUCCESS);
1344 }
1345
1346 void
1347 IoDisconnectInterrupt(iobj)
1348         kinterrupt              *iobj;
1349 {
1350         uint8_t                 irql;
1351
1352         if (iobj == NULL)
1353                 return;
1354
1355         KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1356         RemoveEntryList((&iobj->ki_list));
1357         KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1358
1359         ExFreePool(iobj);
1360 }
1361
1362 device_object *
1363 IoAttachDeviceToDeviceStack(src, dst)
1364         device_object           *src;
1365         device_object           *dst;
1366 {
1367         device_object           *attached;
1368
1369         mtx_lock(&ntoskrnl_dispatchlock);
1370         attached = IoGetAttachedDevice(dst);
1371         attached->do_attacheddev = src;
1372         src->do_attacheddev = NULL;
1373         src->do_stacksize = attached->do_stacksize + 1;
1374         mtx_unlock(&ntoskrnl_dispatchlock);
1375
1376         return (attached);
1377 }
1378
1379 void
1380 IoDetachDevice(topdev)
1381         device_object           *topdev;
1382 {
1383         device_object           *tail;
1384
1385         mtx_lock(&ntoskrnl_dispatchlock);
1386
1387         /* First, break the chain. */
1388         tail = topdev->do_attacheddev;
1389         if (tail == NULL) {
1390                 mtx_unlock(&ntoskrnl_dispatchlock);
1391                 return;
1392         }
1393         topdev->do_attacheddev = tail->do_attacheddev;
1394         topdev->do_refcnt--;
1395
1396         /* Now reduce the stacksize count for the takm_il objects. */
1397
1398         tail = topdev->do_attacheddev;
1399         while (tail != NULL) {
1400                 tail->do_stacksize--;
1401                 tail = tail->do_attacheddev;
1402         }
1403
1404         mtx_unlock(&ntoskrnl_dispatchlock);
1405 }
1406
1407 /*
1408  * For the most part, an object is considered signalled if
1409  * dh_sigstate == TRUE. The exception is for mutant objects
1410  * (mutexes), where the logic works like this:
1411  *
1412  * - If the thread already owns the object and sigstate is
1413  *   less than or equal to 0, then the object is considered
1414  *   signalled (recursive acquisition).
1415  * - If dh_sigstate == 1, the object is also considered
1416  *   signalled.
1417  */
1418
1419 static int
1420 ntoskrnl_is_signalled(obj, td)
1421         nt_dispatch_header      *obj;
1422         struct thread           *td;
1423 {
1424         kmutant                 *km;
1425
1426         if (obj->dh_type == DISP_TYPE_MUTANT) {
1427                 km = (kmutant *)obj;
1428                 if ((obj->dh_sigstate <= 0 && km->km_ownerthread == td) ||
1429                     obj->dh_sigstate == 1)
1430                         return (TRUE);
1431                 return (FALSE);
1432         }
1433
1434         if (obj->dh_sigstate > 0)
1435                 return (TRUE);
1436         return (FALSE);
1437 }
1438
1439 static void
1440 ntoskrnl_satisfy_wait(obj, td)
1441         nt_dispatch_header      *obj;
1442         struct thread           *td;
1443 {
1444         kmutant                 *km;
1445
1446         switch (obj->dh_type) {
1447         case DISP_TYPE_MUTANT:
1448                 km = (struct kmutant *)obj;
1449                 obj->dh_sigstate--;
1450                 /*
1451                  * If sigstate reaches 0, the mutex is now
1452                  * non-signalled (the new thread owns it).
1453                  */
1454                 if (obj->dh_sigstate == 0) {
1455                         km->km_ownerthread = td;
1456                         if (km->km_abandoned == TRUE)
1457                                 km->km_abandoned = FALSE;
1458                 }
1459                 break;
1460         /* Synchronization objects get reset to unsignalled. */
1461         case DISP_TYPE_SYNCHRONIZATION_EVENT:
1462         case DISP_TYPE_SYNCHRONIZATION_TIMER:
1463                 obj->dh_sigstate = 0;
1464                 break;
1465         case DISP_TYPE_SEMAPHORE:
1466                 obj->dh_sigstate--;
1467                 break;
1468         default:
1469                 break;
1470         }
1471 }
1472
1473 static void
1474 ntoskrnl_satisfy_multiple_waits(wb)
1475         wait_block              *wb;
1476 {
1477         wait_block              *cur;
1478         struct thread           *td;
1479
1480         cur = wb;
1481         td = wb->wb_kthread;
1482
1483         do {
1484                 ntoskrnl_satisfy_wait(wb->wb_object, td);
1485                 cur->wb_awakened = TRUE;
1486                 cur = cur->wb_next;
1487         } while (cur != wb);
1488 }
1489
1490 /* Always called with dispatcher lock held. */
1491 static void
1492 ntoskrnl_waittest(obj, increment)
1493         nt_dispatch_header      *obj;
1494         uint32_t                increment;
1495 {
1496         wait_block              *w, *next;
1497         list_entry              *e;
1498         struct thread           *td;
1499         wb_ext                  *we;
1500         int                     satisfied;
1501
1502         /*
1503          * Once an object has been signalled, we walk its list of
1504          * wait blocks. If a wait block can be awakened, then satisfy
1505          * waits as necessary and wake the thread.
1506          *
1507          * The rules work like this:
1508          *
1509          * If a wait block is marked as WAITTYPE_ANY, then
1510          * we can satisfy the wait conditions on the current
1511          * object and wake the thread right away. Satisfying
1512          * the wait also has the effect of breaking us out
1513          * of the search loop.
1514          *
1515          * If the object is marked as WAITTYLE_ALL, then the
1516          * wait block will be part of a circularly linked
1517          * list of wait blocks belonging to a waiting thread
1518          * that's sleeping in KeWaitForMultipleObjects(). In
1519          * order to wake the thread, all the objects in the
1520          * wait list must be in the signalled state. If they
1521          * are, we then satisfy all of them and wake the
1522          * thread.
1523          *
1524          */
1525
1526         e = obj->dh_waitlisthead.nle_flink;
1527
1528         while (e != &obj->dh_waitlisthead && obj->dh_sigstate > 0) {
1529                 w = CONTAINING_RECORD(e, wait_block, wb_waitlist);
1530                 we = w->wb_ext;
1531                 td = we->we_td;
1532                 satisfied = FALSE;
1533                 if (w->wb_waittype == WAITTYPE_ANY) {
1534                         /*
1535                          * Thread can be awakened if
1536                          * any wait is satisfied.
1537                          */
1538                         ntoskrnl_satisfy_wait(obj, td);
1539                         satisfied = TRUE;
1540                         w->wb_awakened = TRUE;
1541                 } else {
1542                         /*
1543                          * Thread can only be woken up
1544                          * if all waits are satisfied.
1545                          * If the thread is waiting on multiple
1546                          * objects, they should all be linked
1547                          * through the wb_next pointers in the
1548                          * wait blocks.
1549                          */
1550                         satisfied = TRUE;
1551                         next = w->wb_next;
1552                         while (next != w) {
1553                                 if (ntoskrnl_is_signalled(obj, td) == FALSE) {
1554                                         satisfied = FALSE;
1555                                         break;
1556                                 }
1557                                 next = next->wb_next;
1558                         }
1559                         ntoskrnl_satisfy_multiple_waits(w);
1560                 }
1561
1562                 if (satisfied == TRUE)
1563                         cv_broadcastpri(&we->we_cv,
1564                             (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
1565                             w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
1566
1567                 e = e->nle_flink;
1568         }
1569 }
1570
1571 /*
1572  * Return the number of 100 nanosecond intervals since
1573  * January 1, 1601. (?!?!)
1574  */
1575 void
1576 ntoskrnl_time(tval)
1577         uint64_t                *tval;
1578 {
1579         struct timespec         ts;
1580
1581         nanotime(&ts);
1582         *tval = (uint64_t)ts.tv_nsec / 100 + (uint64_t)ts.tv_sec * 10000000 +
1583             11644473600 * 10000000; /* 100ns ticks from 1601 to 1970 */
1584 }
1585
1586 static void
1587 KeQuerySystemTime(current_time)
1588         uint64_t                *current_time;
1589 {
1590         ntoskrnl_time(current_time);
1591 }
1592
1593 static uint32_t
1594 KeTickCount(void)
1595 {
1596         struct timeval tv;
1597         getmicrouptime(&tv);
1598         return tvtohz(&tv);
1599 }
1600
1601 /*
1602  * KeWaitForSingleObject() is a tricky beast, because it can be used
1603  * with several different object types: semaphores, timers, events,
1604  * mutexes and threads. Semaphores don't appear very often, but the
1605  * other object types are quite common. KeWaitForSingleObject() is
1606  * what's normally used to acquire a mutex, and it can be used to
1607  * wait for a thread termination.
1608  *
1609  * The Windows NDIS API is implemented in terms of Windows kernel
1610  * primitives, and some of the object manipulation is duplicated in
1611  * NDIS. For example, NDIS has timers and events, which are actually
1612  * Windows kevents and ktimers. Now, you're supposed to only use the
1613  * NDIS variants of these objects within the confines of the NDIS API,
1614  * but there are some naughty developers out there who will use
1615  * KeWaitForSingleObject() on NDIS timer and event objects, so we
1616  * have to support that as well. Conseqently, our NDIS timer and event
1617  * code has to be closely tied into our ntoskrnl timer and event code,
1618  * just as it is in Windows.
1619  *
1620  * KeWaitForSingleObject() may do different things for different kinds
1621  * of objects:
1622  *
1623  * - For events, we check if the event has been signalled. If the
1624  *   event is already in the signalled state, we just return immediately,
1625  *   otherwise we wait for it to be set to the signalled state by someone
1626  *   else calling KeSetEvent(). Events can be either synchronization or
1627  *   notification events.
1628  *
1629  * - For timers, if the timer has already fired and the timer is in
1630  *   the signalled state, we just return, otherwise we wait on the
1631  *   timer. Unlike an event, timers get signalled automatically when
1632  *   they expire rather than someone having to trip them manually.
1633  *   Timers initialized with KeInitializeTimer() are always notification
1634  *   events: KeInitializeTimerEx() lets you initialize a timer as
1635  *   either a notification or synchronization event.
1636  *
1637  * - For mutexes, we try to acquire the mutex and if we can't, we wait
1638  *   on the mutex until it's available and then grab it. When a mutex is
1639  *   released, it enters the signalled state, which wakes up one of the
1640  *   threads waiting to acquire it. Mutexes are always synchronization
1641  *   events.
1642  *
1643  * - For threads, the only thing we do is wait until the thread object
1644  *   enters a signalled state, which occurs when the thread terminates.
1645  *   Threads are always notification events.
1646  *
1647  * A notification event wakes up all threads waiting on an object. A
1648  * synchronization event wakes up just one. Also, a synchronization event
1649  * is auto-clearing, which means we automatically set the event back to
1650  * the non-signalled state once the wakeup is done.
1651  */
1652
1653 uint32_t
1654 KeWaitForSingleObject(void *arg, uint32_t reason, uint32_t mode,
1655     uint8_t alertable, int64_t *duetime)
1656 {
1657         wait_block              w;
1658         struct thread           *td = curthread;
1659         struct timeval          tv;
1660         int                     error = 0;
1661         uint64_t                curtime;
1662         wb_ext                  we;
1663         nt_dispatch_header      *obj;
1664
1665         obj = arg;
1666
1667         if (obj == NULL)
1668                 return (STATUS_INVALID_PARAMETER);
1669
1670         mtx_lock(&ntoskrnl_dispatchlock);
1671
1672         cv_init(&we.we_cv, "KeWFS");
1673         we.we_td = td;
1674
1675         /*
1676          * Check to see if this object is already signalled,
1677          * and just return without waiting if it is.
1678          */
1679         if (ntoskrnl_is_signalled(obj, td) == TRUE) {
1680                 /* Sanity check the signal state value. */
1681                 if (obj->dh_sigstate != INT32_MIN) {
1682                         ntoskrnl_satisfy_wait(obj, curthread);
1683                         mtx_unlock(&ntoskrnl_dispatchlock);
1684                         return (STATUS_SUCCESS);
1685                 } else {
1686                         /*
1687                          * There's a limit to how many times we can
1688                          * recursively acquire a mutant. If we hit
1689                          * the limit, something is very wrong.
1690                          */
1691                         if (obj->dh_type == DISP_TYPE_MUTANT) {
1692                                 mtx_unlock(&ntoskrnl_dispatchlock);
1693                                 panic("mutant limit exceeded");
1694                         }
1695                 }
1696         }
1697
1698         bzero((char *)&w, sizeof(wait_block));
1699         w.wb_object = obj;
1700         w.wb_ext = &we;
1701         w.wb_waittype = WAITTYPE_ANY;
1702         w.wb_next = &w;
1703         w.wb_waitkey = 0;
1704         w.wb_awakened = FALSE;
1705         w.wb_oldpri = td->td_priority;
1706
1707         InsertTailList((&obj->dh_waitlisthead), (&w.wb_waitlist));
1708
1709         /*
1710          * The timeout value is specified in 100 nanosecond units
1711          * and can be a positive or negative number. If it's positive,
1712          * then the duetime is absolute, and we need to convert it
1713          * to an absolute offset relative to now in order to use it.
1714          * If it's negative, then the duetime is relative and we
1715          * just have to convert the units.
1716          */
1717
1718         if (duetime != NULL) {
1719                 if (*duetime < 0) {
1720                         tv.tv_sec = - (*duetime) / 10000000;
1721                         tv.tv_usec = (- (*duetime) / 10) -
1722                             (tv.tv_sec * 1000000);
1723                 } else {
1724                         ntoskrnl_time(&curtime);
1725                         if (*duetime < curtime)
1726                                 tv.tv_sec = tv.tv_usec = 0;
1727                         else {
1728                                 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1729                                 tv.tv_usec = ((*duetime) - curtime) / 10 -
1730                                     (tv.tv_sec * 1000000);
1731                         }
1732                 }
1733         }
1734
1735         if (duetime == NULL)
1736                 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1737         else
1738                 error = cv_timedwait(&we.we_cv,
1739                     &ntoskrnl_dispatchlock, tvtohz(&tv));
1740
1741         RemoveEntryList(&w.wb_waitlist);
1742
1743         cv_destroy(&we.we_cv);
1744
1745         /* We timed out. Leave the object alone and return status. */
1746
1747         if (error == EWOULDBLOCK) {
1748                 mtx_unlock(&ntoskrnl_dispatchlock);
1749                 return (STATUS_TIMEOUT);
1750         }
1751
1752         mtx_unlock(&ntoskrnl_dispatchlock);
1753
1754         return (STATUS_SUCCESS);
1755 /*
1756         return (KeWaitForMultipleObjects(1, &obj, WAITTYPE_ALL, reason,
1757             mode, alertable, duetime, &w));
1758 */
1759 }
1760
1761 static uint32_t
1762 KeWaitForMultipleObjects(uint32_t cnt, nt_dispatch_header *obj[], uint32_t wtype,
1763         uint32_t reason, uint32_t mode, uint8_t alertable, int64_t *duetime,
1764         wait_block *wb_array)
1765 {
1766         struct thread           *td = curthread;
1767         wait_block              *whead, *w;
1768         wait_block              _wb_array[MAX_WAIT_OBJECTS];
1769         nt_dispatch_header      *cur;
1770         struct timeval          tv;
1771         int                     i, wcnt = 0, error = 0;
1772         uint64_t                curtime;
1773         struct timespec         t1, t2;
1774         uint32_t                status = STATUS_SUCCESS;
1775         wb_ext                  we;
1776
1777         if (cnt > MAX_WAIT_OBJECTS)
1778                 return (STATUS_INVALID_PARAMETER);
1779         if (cnt > THREAD_WAIT_OBJECTS && wb_array == NULL)
1780                 return (STATUS_INVALID_PARAMETER);
1781
1782         mtx_lock(&ntoskrnl_dispatchlock);
1783
1784         cv_init(&we.we_cv, "KeWFM");
1785         we.we_td = td;
1786
1787         if (wb_array == NULL)
1788                 whead = _wb_array;
1789         else
1790                 whead = wb_array;
1791
1792         bzero((char *)whead, sizeof(wait_block) * cnt);
1793
1794         /* First pass: see if we can satisfy any waits immediately. */
1795
1796         wcnt = 0;
1797         w = whead;
1798
1799         for (i = 0; i < cnt; i++) {
1800                 InsertTailList((&obj[i]->dh_waitlisthead),
1801                     (&w->wb_waitlist));
1802                 w->wb_ext = &we;
1803                 w->wb_object = obj[i];
1804                 w->wb_waittype = wtype;
1805                 w->wb_waitkey = i;
1806                 w->wb_awakened = FALSE;
1807                 w->wb_oldpri = td->td_priority;
1808                 w->wb_next = w + 1;
1809                 w++;
1810                 wcnt++;
1811                 if (ntoskrnl_is_signalled(obj[i], td)) {
1812                         /*
1813                          * There's a limit to how many times
1814                          * we can recursively acquire a mutant.
1815                          * If we hit the limit, something
1816                          * is very wrong.
1817                          */
1818                         if (obj[i]->dh_sigstate == INT32_MIN &&
1819                             obj[i]->dh_type == DISP_TYPE_MUTANT) {
1820                                 mtx_unlock(&ntoskrnl_dispatchlock);
1821                                 panic("mutant limit exceeded");
1822                         }
1823
1824                         /*
1825                          * If this is a WAITTYPE_ANY wait, then
1826                          * satisfy the waited object and exit
1827                          * right now.
1828                          */
1829
1830                         if (wtype == WAITTYPE_ANY) {
1831                                 ntoskrnl_satisfy_wait(obj[i], td);
1832                                 status = STATUS_WAIT_0 + i;
1833                                 goto wait_done;
1834                         } else {
1835                                 w--;
1836                                 wcnt--;
1837                                 w->wb_object = NULL;
1838                                 RemoveEntryList(&w->wb_waitlist);
1839                         }
1840                 }
1841         }
1842
1843         /*
1844          * If this is a WAITTYPE_ALL wait and all objects are
1845          * already signalled, satisfy the waits and exit now.
1846          */
1847
1848         if (wtype == WAITTYPE_ALL && wcnt == 0) {
1849                 for (i = 0; i < cnt; i++)
1850                         ntoskrnl_satisfy_wait(obj[i], td);
1851                 status = STATUS_SUCCESS;
1852                 goto wait_done;
1853         }
1854
1855         /*
1856          * Create a circular waitblock list. The waitcount
1857          * must always be non-zero when we get here.
1858          */
1859
1860         (w - 1)->wb_next = whead;
1861
1862         /* Wait on any objects that aren't yet signalled. */
1863
1864         /* Calculate timeout, if any. */
1865
1866         if (duetime != NULL) {
1867                 if (*duetime < 0) {
1868                         tv.tv_sec = - (*duetime) / 10000000;
1869                         tv.tv_usec = (- (*duetime) / 10) -
1870                             (tv.tv_sec * 1000000);
1871                 } else {
1872                         ntoskrnl_time(&curtime);
1873                         if (*duetime < curtime)
1874                                 tv.tv_sec = tv.tv_usec = 0;
1875                         else {
1876                                 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1877                                 tv.tv_usec = ((*duetime) - curtime) / 10 -
1878                                     (tv.tv_sec * 1000000);
1879                         }
1880                 }
1881         }
1882
1883         while (wcnt) {
1884                 nanotime(&t1);
1885
1886                 if (duetime == NULL)
1887                         cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1888                 else
1889                         error = cv_timedwait(&we.we_cv,
1890                             &ntoskrnl_dispatchlock, tvtohz(&tv));
1891
1892                 /* Wait with timeout expired. */
1893
1894                 if (error) {
1895                         status = STATUS_TIMEOUT;
1896                         goto wait_done;
1897                 }
1898
1899                 nanotime(&t2);
1900
1901                 /* See what's been signalled. */
1902
1903                 w = whead;
1904                 do {
1905                         cur = w->wb_object;
1906                         if (ntoskrnl_is_signalled(cur, td) == TRUE ||
1907                             w->wb_awakened == TRUE) {
1908                                 /* Sanity check the signal state value. */
1909                                 if (cur->dh_sigstate == INT32_MIN &&
1910                                     cur->dh_type == DISP_TYPE_MUTANT) {
1911                                         mtx_unlock(&ntoskrnl_dispatchlock);
1912                                         panic("mutant limit exceeded");
1913                                 }
1914                                 wcnt--;
1915                                 if (wtype == WAITTYPE_ANY) {
1916                                         status = w->wb_waitkey &
1917                                             STATUS_WAIT_0;
1918                                         goto wait_done;
1919                                 }
1920                         }
1921                         w = w->wb_next;
1922                 } while (w != whead);
1923
1924                 /*
1925                  * If all objects have been signalled, or if this
1926                  * is a WAITTYPE_ANY wait and we were woke up by
1927                  * someone, we can bail.
1928                  */
1929
1930                 if (wcnt == 0) {
1931                         status = STATUS_SUCCESS;
1932                         goto wait_done;
1933                 }
1934
1935                 /*
1936                  * If this is WAITTYPE_ALL wait, and there's still
1937                  * objects that haven't been signalled, deduct the
1938                  * time that's elapsed so far from the timeout and
1939                  * wait again (or continue waiting indefinitely if
1940                  * there's no timeout).
1941                  */
1942
1943                 if (duetime != NULL) {
1944                         tv.tv_sec -= (t2.tv_sec - t1.tv_sec);
1945                         tv.tv_usec -= (t2.tv_nsec - t1.tv_nsec) / 1000;
1946                 }
1947         }
1948
1949 wait_done:
1950
1951         cv_destroy(&we.we_cv);
1952
1953         for (i = 0; i < cnt; i++) {
1954                 if (whead[i].wb_object != NULL)
1955                         RemoveEntryList(&whead[i].wb_waitlist);
1956         }
1957         mtx_unlock(&ntoskrnl_dispatchlock);
1958
1959         return (status);
1960 }
1961
1962 static void
1963 WRITE_REGISTER_USHORT(uint16_t *reg, uint16_t val)
1964 {
1965         bus_space_write_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1966 }
1967
1968 static uint16_t
1969 READ_REGISTER_USHORT(reg)
1970         uint16_t                *reg;
1971 {
1972         return (bus_space_read_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1973 }
1974
1975 static void
1976 WRITE_REGISTER_ULONG(reg, val)
1977         uint32_t                *reg;
1978         uint32_t                val;
1979 {
1980         bus_space_write_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1981 }
1982
1983 static uint32_t
1984 READ_REGISTER_ULONG(reg)
1985         uint32_t                *reg;
1986 {
1987         return (bus_space_read_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1988 }
1989
1990 static uint8_t
1991 READ_REGISTER_UCHAR(uint8_t *reg)
1992 {
1993         return (bus_space_read_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1994 }
1995
1996 static void
1997 WRITE_REGISTER_UCHAR(uint8_t *reg, uint8_t val)
1998 {
1999         bus_space_write_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
2000 }
2001
2002 static int64_t
2003 _allmul(a, b)
2004         int64_t                 a;
2005         int64_t                 b;
2006 {
2007         return (a * b);
2008 }
2009
2010 static int64_t
2011 _alldiv(a, b)
2012         int64_t                 a;
2013         int64_t                 b;
2014 {
2015         return (a / b);
2016 }
2017
2018 static int64_t
2019 _allrem(a, b)
2020         int64_t                 a;
2021         int64_t                 b;
2022 {
2023         return (a % b);
2024 }
2025
2026 static uint64_t
2027 _aullmul(a, b)
2028         uint64_t                a;
2029         uint64_t                b;
2030 {
2031         return (a * b);
2032 }
2033
2034 static uint64_t
2035 _aulldiv(a, b)
2036         uint64_t                a;
2037         uint64_t                b;
2038 {
2039         return (a / b);
2040 }
2041
2042 static uint64_t
2043 _aullrem(a, b)
2044         uint64_t                a;
2045         uint64_t                b;
2046 {
2047         return (a % b);
2048 }
2049
2050 static int64_t
2051 _allshl(int64_t a, uint8_t b)
2052 {
2053         return (a << b);
2054 }
2055
2056 static uint64_t
2057 _aullshl(uint64_t a, uint8_t b)
2058 {
2059         return (a << b);
2060 }
2061
2062 static int64_t
2063 _allshr(int64_t a, uint8_t b)
2064 {
2065         return (a >> b);
2066 }
2067
2068 static uint64_t
2069 _aullshr(uint64_t a, uint8_t b)
2070 {
2071         return (a >> b);
2072 }
2073
2074 static slist_entry *
2075 ntoskrnl_pushsl(head, entry)
2076         slist_header            *head;
2077         slist_entry             *entry;
2078 {
2079         slist_entry             *oldhead;
2080
2081         oldhead = head->slh_list.slh_next;
2082         entry->sl_next = head->slh_list.slh_next;
2083         head->slh_list.slh_next = entry;
2084         head->slh_list.slh_depth++;
2085         head->slh_list.slh_seq++;
2086
2087         return (oldhead);
2088 }
2089
2090 static void
2091 InitializeSListHead(head)
2092         slist_header            *head;
2093 {
2094         memset(head, 0, sizeof(*head));
2095 }
2096
2097 static slist_entry *
2098 ntoskrnl_popsl(head)
2099         slist_header            *head;
2100 {
2101         slist_entry             *first;
2102
2103         first = head->slh_list.slh_next;
2104         if (first != NULL) {
2105                 head->slh_list.slh_next = first->sl_next;
2106                 head->slh_list.slh_depth--;
2107                 head->slh_list.slh_seq++;
2108         }
2109
2110         return (first);
2111 }
2112
2113 /*
2114  * We need this to make lookaside lists work for amd64.
2115  * We pass a pointer to ExAllocatePoolWithTag() the lookaside
2116  * list structure. For amd64 to work right, this has to be a
2117  * pointer to the wrapped version of the routine, not the
2118  * original. Letting the Windows driver invoke the original
2119  * function directly will result in a convention calling
2120  * mismatch and a pretty crash. On x86, this effectively
2121  * becomes a no-op since ipt_func and ipt_wrap are the same.
2122  */
2123
2124 static funcptr
2125 ntoskrnl_findwrap(func)
2126         funcptr                 func;
2127 {
2128         image_patch_table       *patch;
2129
2130         patch = ntoskrnl_functbl;
2131         while (patch->ipt_func != NULL) {
2132                 if ((funcptr)patch->ipt_func == func)
2133                         return ((funcptr)patch->ipt_wrap);
2134                 patch++;
2135         }
2136
2137         return (NULL);
2138 }
2139
2140 static void
2141 ExInitializePagedLookasideList(paged_lookaside_list *lookaside,
2142         lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2143         uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2144 {
2145         bzero((char *)lookaside, sizeof(paged_lookaside_list));
2146
2147         if (size < sizeof(slist_entry))
2148                 lookaside->nll_l.gl_size = sizeof(slist_entry);
2149         else
2150                 lookaside->nll_l.gl_size = size;
2151         lookaside->nll_l.gl_tag = tag;
2152         if (allocfunc == NULL)
2153                 lookaside->nll_l.gl_allocfunc =
2154                     ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2155         else
2156                 lookaside->nll_l.gl_allocfunc = allocfunc;
2157
2158         if (freefunc == NULL)
2159                 lookaside->nll_l.gl_freefunc =
2160                     ntoskrnl_findwrap((funcptr)ExFreePool);
2161         else
2162                 lookaside->nll_l.gl_freefunc = freefunc;
2163
2164 #ifdef __i386__
2165         KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2166 #endif
2167
2168         lookaside->nll_l.gl_type = NonPagedPool;
2169         lookaside->nll_l.gl_depth = depth;
2170         lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2171 }
2172
2173 static void
2174 ExDeletePagedLookasideList(lookaside)
2175         paged_lookaside_list   *lookaside;
2176 {
2177         void                    *buf;
2178         void            (*freefunc)(void *);
2179
2180         freefunc = lookaside->nll_l.gl_freefunc;
2181         while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2182                 MSCALL1(freefunc, buf);
2183 }
2184
2185 static void
2186 ExInitializeNPagedLookasideList(npaged_lookaside_list *lookaside,
2187         lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2188         uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2189 {
2190         bzero((char *)lookaside, sizeof(npaged_lookaside_list));
2191
2192         if (size < sizeof(slist_entry))
2193                 lookaside->nll_l.gl_size = sizeof(slist_entry);
2194         else
2195                 lookaside->nll_l.gl_size = size;
2196         lookaside->nll_l.gl_tag = tag;
2197         if (allocfunc == NULL)
2198                 lookaside->nll_l.gl_allocfunc =
2199                     ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2200         else
2201                 lookaside->nll_l.gl_allocfunc = allocfunc;
2202
2203         if (freefunc == NULL)
2204                 lookaside->nll_l.gl_freefunc =
2205                     ntoskrnl_findwrap((funcptr)ExFreePool);
2206         else
2207                 lookaside->nll_l.gl_freefunc = freefunc;
2208
2209 #ifdef __i386__
2210         KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2211 #endif
2212
2213         lookaside->nll_l.gl_type = NonPagedPool;
2214         lookaside->nll_l.gl_depth = depth;
2215         lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2216 }
2217
2218 static void
2219 ExDeleteNPagedLookasideList(lookaside)
2220         npaged_lookaside_list   *lookaside;
2221 {
2222         void                    *buf;
2223         void            (*freefunc)(void *);
2224
2225         freefunc = lookaside->nll_l.gl_freefunc;
2226         while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2227                 MSCALL1(freefunc, buf);
2228 }
2229
2230 slist_entry *
2231 InterlockedPushEntrySList(head, entry)
2232         slist_header            *head;
2233         slist_entry             *entry;
2234 {
2235         slist_entry             *oldhead;
2236
2237         mtx_lock_spin(&ntoskrnl_interlock);
2238         oldhead = ntoskrnl_pushsl(head, entry);
2239         mtx_unlock_spin(&ntoskrnl_interlock);
2240
2241         return (oldhead);
2242 }
2243
2244 slist_entry *
2245 InterlockedPopEntrySList(head)
2246         slist_header            *head;
2247 {
2248         slist_entry             *first;
2249
2250         mtx_lock_spin(&ntoskrnl_interlock);
2251         first = ntoskrnl_popsl(head);
2252         mtx_unlock_spin(&ntoskrnl_interlock);
2253
2254         return (first);
2255 }
2256
2257 static slist_entry *
2258 ExInterlockedPushEntrySList(head, entry, lock)
2259         slist_header            *head;
2260         slist_entry             *entry;
2261         kspin_lock              *lock;
2262 {
2263         return (InterlockedPushEntrySList(head, entry));
2264 }
2265
2266 static slist_entry *
2267 ExInterlockedPopEntrySList(head, lock)
2268         slist_header            *head;
2269         kspin_lock              *lock;
2270 {
2271         return (InterlockedPopEntrySList(head));
2272 }
2273
2274 uint16_t
2275 ExQueryDepthSList(head)
2276         slist_header            *head;
2277 {
2278         uint16_t                depth;
2279
2280         mtx_lock_spin(&ntoskrnl_interlock);
2281         depth = head->slh_list.slh_depth;
2282         mtx_unlock_spin(&ntoskrnl_interlock);
2283
2284         return (depth);
2285 }
2286
2287 void
2288 KeInitializeSpinLock(lock)
2289         kspin_lock              *lock;
2290 {
2291         *lock = 0;
2292 }
2293
2294 #ifdef __i386__
2295 void
2296 KefAcquireSpinLockAtDpcLevel(lock)
2297         kspin_lock              *lock;
2298 {
2299 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2300         int                     i = 0;
2301 #endif
2302
2303         while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) {
2304                 /* sit and spin */;
2305 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2306                 i++;
2307                 if (i > 200000000)
2308                         panic("DEADLOCK!");
2309 #endif
2310         }
2311 }
2312
2313 void
2314 KefReleaseSpinLockFromDpcLevel(lock)
2315         kspin_lock              *lock;
2316 {
2317         atomic_store_rel_int((volatile u_int *)lock, 0);
2318 }
2319
2320 uint8_t
2321 KeAcquireSpinLockRaiseToDpc(kspin_lock *lock)
2322 {
2323         uint8_t                 oldirql;
2324
2325         if (KeGetCurrentIrql() > DISPATCH_LEVEL)
2326                 panic("IRQL_NOT_LESS_THAN_OR_EQUAL");
2327
2328         KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
2329         KeAcquireSpinLockAtDpcLevel(lock);
2330
2331         return (oldirql);
2332 }
2333 #else
2334 void
2335 KeAcquireSpinLockAtDpcLevel(kspin_lock *lock)
2336 {
2337         while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0)
2338                 /* sit and spin */;
2339 }
2340
2341 void
2342 KeReleaseSpinLockFromDpcLevel(kspin_lock *lock)
2343 {
2344         atomic_store_rel_int((volatile u_int *)lock, 0);
2345 }
2346 #endif /* __i386__ */
2347
2348 uintptr_t
2349 InterlockedExchange(dst, val)
2350         volatile uint32_t       *dst;
2351         uintptr_t               val;
2352 {
2353         uintptr_t               r;
2354
2355         mtx_lock_spin(&ntoskrnl_interlock);
2356         r = *dst;
2357         *dst = val;
2358         mtx_unlock_spin(&ntoskrnl_interlock);
2359
2360         return (r);
2361 }
2362
2363 static uint32_t
2364 InterlockedIncrement(addend)
2365         volatile uint32_t       *addend;
2366 {
2367         atomic_add_long((volatile u_long *)addend, 1);
2368         return (*addend);
2369 }
2370
2371 static uint32_t
2372 InterlockedDecrement(addend)
2373         volatile uint32_t       *addend;
2374 {
2375         atomic_subtract_long((volatile u_long *)addend, 1);
2376         return (*addend);
2377 }
2378
2379 static void
2380 ExInterlockedAddLargeStatistic(addend, inc)
2381         uint64_t                *addend;
2382         uint32_t                inc;
2383 {
2384         mtx_lock_spin(&ntoskrnl_interlock);
2385         *addend += inc;
2386         mtx_unlock_spin(&ntoskrnl_interlock);
2387 };
2388
2389 mdl *
2390 IoAllocateMdl(void *vaddr, uint32_t len, uint8_t secondarybuf,
2391         uint8_t chargequota, irp *iopkt)
2392 {
2393         mdl                     *m;
2394         int                     zone = 0;
2395
2396         if (MmSizeOfMdl(vaddr, len) > MDL_ZONE_SIZE)
2397                 m = ExAllocatePoolWithTag(NonPagedPool,
2398                     MmSizeOfMdl(vaddr, len), 0);
2399         else {
2400                 m = uma_zalloc(mdl_zone, M_NOWAIT | M_ZERO);
2401                 zone++;
2402         }
2403
2404         if (m == NULL)
2405                 return (NULL);
2406
2407         MmInitializeMdl(m, vaddr, len);
2408
2409         /*
2410          * MmInitializMdl() clears the flags field, so we
2411          * have to set this here. If the MDL came from the
2412          * MDL UMA zone, tag it so we can release it to
2413          * the right place later.
2414          */
2415         if (zone)
2416                 m->mdl_flags = MDL_ZONE_ALLOCED;
2417
2418         if (iopkt != NULL) {
2419                 if (secondarybuf == TRUE) {
2420                         mdl                     *last;
2421                         last = iopkt->irp_mdl;
2422                         while (last->mdl_next != NULL)
2423                                 last = last->mdl_next;
2424                         last->mdl_next = m;
2425                 } else {
2426                         if (iopkt->irp_mdl != NULL)
2427                                 panic("leaking an MDL in IoAllocateMdl()");
2428                         iopkt->irp_mdl = m;
2429                 }
2430         }
2431
2432         return (m);
2433 }
2434
2435 void
2436 IoFreeMdl(m)
2437         mdl                     *m;
2438 {
2439         if (m == NULL)
2440                 return;
2441
2442         if (m->mdl_flags & MDL_ZONE_ALLOCED)
2443                 uma_zfree(mdl_zone, m);
2444         else
2445                 ExFreePool(m);
2446 }
2447
2448 static void *
2449 MmAllocateContiguousMemory(size, highest)
2450         uint32_t                size;
2451         uint64_t                highest;
2452 {
2453         void *addr;
2454         size_t pagelength = roundup(size, PAGE_SIZE);
2455
2456         addr = ExAllocatePoolWithTag(NonPagedPool, pagelength, 0);
2457
2458         return (addr);
2459 }
2460
2461 static void *
2462 MmAllocateContiguousMemorySpecifyCache(size, lowest, highest,
2463     boundary, cachetype)
2464         uint32_t                size;
2465         uint64_t                lowest;
2466         uint64_t                highest;
2467         uint64_t                boundary;
2468         enum nt_caching_type    cachetype;
2469 {
2470         vm_memattr_t            memattr;
2471         void                    *ret;
2472
2473         switch (cachetype) {
2474         case MmNonCached:
2475                 memattr = VM_MEMATTR_UNCACHEABLE;
2476                 break;
2477         case MmWriteCombined:
2478                 memattr = VM_MEMATTR_WRITE_COMBINING;
2479                 break;
2480         case MmNonCachedUnordered:
2481                 memattr = VM_MEMATTR_UNCACHEABLE;
2482                 break;
2483         case MmCached:
2484         case MmHardwareCoherentCached:
2485         case MmUSWCCached:
2486         default:
2487                 memattr = VM_MEMATTR_DEFAULT;
2488                 break;
2489         }
2490
2491         ret = (void *)kmem_alloc_contig(size, M_ZERO | M_NOWAIT, lowest,
2492             highest, PAGE_SIZE, boundary, memattr);
2493         if (ret != NULL)
2494                 malloc_type_allocated(M_DEVBUF, round_page(size));
2495         return (ret);
2496 }
2497
2498 static void
2499 MmFreeContiguousMemory(base)
2500         void                    *base;
2501 {
2502         ExFreePool(base);
2503 }
2504
2505 static void
2506 MmFreeContiguousMemorySpecifyCache(base, size, cachetype)
2507         void                    *base;
2508         uint32_t                size;
2509         enum nt_caching_type    cachetype;
2510 {
2511         contigfree(base, size, M_DEVBUF);
2512 }
2513
2514 static uint32_t
2515 MmSizeOfMdl(vaddr, len)
2516         void                    *vaddr;
2517         size_t                  len;
2518 {
2519         uint32_t                l;
2520
2521         l = sizeof(struct mdl) +
2522             (sizeof(vm_offset_t *) * SPAN_PAGES(vaddr, len));
2523
2524         return (l);
2525 }
2526
2527 /*
2528  * The Microsoft documentation says this routine fills in the
2529  * page array of an MDL with the _physical_ page addresses that
2530  * comprise the buffer, but we don't really want to do that here.
2531  * Instead, we just fill in the page array with the kernel virtual
2532  * addresses of the buffers.
2533  */
2534 void
2535 MmBuildMdlForNonPagedPool(m)
2536         mdl                     *m;
2537 {
2538         vm_offset_t             *mdl_pages;
2539         int                     pagecnt, i;
2540
2541         pagecnt = SPAN_PAGES(m->mdl_byteoffset, m->mdl_bytecount);
2542
2543         if (pagecnt > (m->mdl_size - sizeof(mdl)) / sizeof(vm_offset_t *))
2544                 panic("not enough pages in MDL to describe buffer");
2545
2546         mdl_pages = MmGetMdlPfnArray(m);
2547
2548         for (i = 0; i < pagecnt; i++)
2549                 *mdl_pages = (vm_offset_t)m->mdl_startva + (i * PAGE_SIZE);
2550
2551         m->mdl_flags |= MDL_SOURCE_IS_NONPAGED_POOL;
2552         m->mdl_mappedsystemva = MmGetMdlVirtualAddress(m);
2553 }
2554
2555 static void *
2556 MmMapLockedPages(mdl *buf, uint8_t accessmode)
2557 {
2558         buf->mdl_flags |= MDL_MAPPED_TO_SYSTEM_VA;
2559         return (MmGetMdlVirtualAddress(buf));
2560 }
2561
2562 static void *
2563 MmMapLockedPagesSpecifyCache(mdl *buf, uint8_t accessmode, uint32_t cachetype,
2564         void *vaddr, uint32_t bugcheck, uint32_t prio)
2565 {
2566         return (MmMapLockedPages(buf, accessmode));
2567 }
2568
2569 static void
2570 MmUnmapLockedPages(vaddr, buf)
2571         void                    *vaddr;
2572         mdl                     *buf;
2573 {
2574         buf->mdl_flags &= ~MDL_MAPPED_TO_SYSTEM_VA;
2575 }
2576
2577 /*
2578  * This function has a problem in that it will break if you
2579  * compile this module without PAE and try to use it on a PAE
2580  * kernel. Unfortunately, there's no way around this at the
2581  * moment. It's slightly less broken that using pmap_kextract().
2582  * You'd think the virtual memory subsystem would help us out
2583  * here, but it doesn't.
2584  */
2585
2586 static uint64_t
2587 MmGetPhysicalAddress(void *base)
2588 {
2589         return (pmap_extract(kernel_map->pmap, (vm_offset_t)base));
2590 }
2591
2592 void *
2593 MmGetSystemRoutineAddress(ustr)
2594         unicode_string          *ustr;
2595 {
2596         ansi_string             astr;
2597
2598         if (RtlUnicodeStringToAnsiString(&astr, ustr, TRUE))
2599                 return (NULL);
2600         return (ndis_get_routine_address(ntoskrnl_functbl, astr.as_buf));
2601 }
2602
2603 uint8_t
2604 MmIsAddressValid(vaddr)
2605         void                    *vaddr;
2606 {
2607         if (pmap_extract(kernel_map->pmap, (vm_offset_t)vaddr))
2608                 return (TRUE);
2609
2610         return (FALSE);
2611 }
2612
2613 void *
2614 MmMapIoSpace(paddr, len, cachetype)
2615         uint64_t                paddr;
2616         uint32_t                len;
2617         uint32_t                cachetype;
2618 {
2619         devclass_t              nexus_class;
2620         device_t                *nexus_devs, devp;
2621         int                     nexus_count = 0;
2622         device_t                matching_dev = NULL;
2623         struct resource         *res;
2624         int                     i;
2625         vm_offset_t             v;
2626
2627         /* There will always be at least one nexus. */
2628
2629         nexus_class = devclass_find("nexus");
2630         devclass_get_devices(nexus_class, &nexus_devs, &nexus_count);
2631
2632         for (i = 0; i < nexus_count; i++) {
2633                 devp = nexus_devs[i];
2634                 matching_dev = ntoskrnl_finddev(devp, paddr, &res);
2635                 if (matching_dev)
2636                         break;
2637         }
2638
2639         free(nexus_devs, M_TEMP);
2640
2641         if (matching_dev == NULL)
2642                 return (NULL);
2643
2644         v = (vm_offset_t)rman_get_virtual(res);
2645         if (paddr > rman_get_start(res))
2646                 v += paddr - rman_get_start(res);
2647
2648         return ((void *)v);
2649 }
2650
2651 void
2652 MmUnmapIoSpace(vaddr, len)
2653         void                    *vaddr;
2654         size_t                  len;
2655 {
2656 }
2657
2658 static device_t
2659 ntoskrnl_finddev(dev, paddr, res)
2660         device_t                dev;
2661         uint64_t                paddr;
2662         struct resource         **res;
2663 {
2664         device_t                *children = NULL;
2665         device_t                matching_dev;
2666         int                     childcnt;
2667         struct resource         *r;
2668         struct resource_list    *rl;
2669         struct resource_list_entry      *rle;
2670         uint32_t                flags;
2671         int                     i;
2672
2673         /* We only want devices that have been successfully probed. */
2674
2675         if (device_is_alive(dev) == FALSE)
2676                 return (NULL);
2677
2678         rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
2679         if (rl != NULL) {
2680                 STAILQ_FOREACH(rle, rl, link) {
2681                         r = rle->res;
2682
2683                         if (r == NULL)
2684                                 continue;
2685
2686                         flags = rman_get_flags(r);
2687
2688                         if (rle->type == SYS_RES_MEMORY &&
2689                             paddr >= rman_get_start(r) &&
2690                             paddr <= rman_get_end(r)) {
2691                                 if (!(flags & RF_ACTIVE))
2692                                         bus_activate_resource(dev,
2693                                             SYS_RES_MEMORY, 0, r);
2694                                 *res = r;
2695                                 return (dev);
2696                         }
2697                 }
2698         }
2699
2700         /*
2701          * If this device has children, do another
2702          * level of recursion to inspect them.
2703          */
2704
2705         device_get_children(dev, &children, &childcnt);
2706
2707         for (i = 0; i < childcnt; i++) {
2708                 matching_dev = ntoskrnl_finddev(children[i], paddr, res);
2709                 if (matching_dev != NULL) {
2710                         free(children, M_TEMP);
2711                         return (matching_dev);
2712                 }
2713         }
2714
2715         /* Won't somebody please think of the children! */
2716
2717         if (children != NULL)
2718                 free(children, M_TEMP);
2719
2720         return (NULL);
2721 }
2722
2723 /*
2724  * Workitems are unlike DPCs, in that they run in a user-mode thread
2725  * context rather than at DISPATCH_LEVEL in kernel context. In our
2726  * case we run them in kernel context anyway.
2727  */
2728 static void
2729 ntoskrnl_workitem_thread(arg)
2730         void                    *arg;
2731 {
2732         kdpc_queue              *kq;
2733         list_entry              *l;
2734         io_workitem             *iw;
2735         uint8_t                 irql;
2736
2737         kq = arg;
2738
2739         InitializeListHead(&kq->kq_disp);
2740         kq->kq_td = curthread;
2741         kq->kq_exit = 0;
2742         KeInitializeSpinLock(&kq->kq_lock);
2743         KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
2744
2745         while (1) {
2746                 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
2747
2748                 KeAcquireSpinLock(&kq->kq_lock, &irql);
2749
2750                 if (kq->kq_exit) {
2751                         kq->kq_exit = 0;
2752                         KeReleaseSpinLock(&kq->kq_lock, irql);
2753                         break;
2754                 }
2755
2756                 while (!IsListEmpty(&kq->kq_disp)) {
2757                         l = RemoveHeadList(&kq->kq_disp);
2758                         iw = CONTAINING_RECORD(l,
2759                             io_workitem, iw_listentry);
2760                         InitializeListHead((&iw->iw_listentry));
2761                         if (iw->iw_func == NULL)
2762                                 continue;
2763                         KeReleaseSpinLock(&kq->kq_lock, irql);
2764                         MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx);
2765                         KeAcquireSpinLock(&kq->kq_lock, &irql);
2766                 }
2767
2768                 KeReleaseSpinLock(&kq->kq_lock, irql);
2769         }
2770
2771         kproc_exit(0);
2772         return; /* notreached */
2773 }
2774
2775 static ndis_status
2776 RtlCharToInteger(src, base, val)
2777         const char              *src;
2778         uint32_t                base;
2779         uint32_t                *val;
2780 {
2781         int negative = 0;
2782         uint32_t res;
2783
2784         if (!src || !val)
2785                 return (STATUS_ACCESS_VIOLATION);
2786         while (*src != '\0' && *src <= ' ')
2787                 src++;
2788         if (*src == '+')
2789                 src++;
2790         else if (*src == '-') {
2791                 src++;
2792                 negative = 1;
2793         }
2794         if (base == 0) {
2795                 base = 10;
2796                 if (*src == '0') {
2797                         src++;
2798                         if (*src == 'b') {
2799                                 base = 2;
2800                                 src++;
2801                         } else if (*src == 'o') {
2802                                 base = 8;
2803                                 src++;
2804                         } else if (*src == 'x') {
2805                                 base = 16;
2806                                 src++;
2807                         }
2808                 }
2809         } else if (!(base == 2 || base == 8 || base == 10 || base == 16))
2810                 return (STATUS_INVALID_PARAMETER);
2811
2812         for (res = 0; *src; src++) {
2813                 int v;
2814                 if (isdigit(*src))
2815                         v = *src - '0';
2816                 else if (isxdigit(*src))
2817                         v = tolower(*src) - 'a' + 10;
2818                 else
2819                         v = base;
2820                 if (v >= base)
2821                         return (STATUS_INVALID_PARAMETER);
2822                 res = res * base + v;
2823         }
2824         *val = negative ? -res : res;
2825         return (STATUS_SUCCESS);
2826 }
2827
2828 static void
2829 ntoskrnl_destroy_workitem_threads(void)
2830 {
2831         kdpc_queue              *kq;
2832         int                     i;
2833
2834         for (i = 0; i < WORKITEM_THREADS; i++) {
2835                 kq = wq_queues + i;
2836                 kq->kq_exit = 1;
2837                 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2838                 while (kq->kq_exit)
2839                         tsleep(kq->kq_td->td_proc, PWAIT, "waitiw", hz/10);
2840         }
2841 }
2842
2843 io_workitem *
2844 IoAllocateWorkItem(dobj)
2845         device_object           *dobj;
2846 {
2847         io_workitem             *iw;
2848
2849         iw = uma_zalloc(iw_zone, M_NOWAIT);
2850         if (iw == NULL)
2851                 return (NULL);
2852
2853         InitializeListHead(&iw->iw_listentry);
2854         iw->iw_dobj = dobj;
2855
2856         mtx_lock(&ntoskrnl_dispatchlock);
2857         iw->iw_idx = wq_idx;
2858         WORKIDX_INC(wq_idx);
2859         mtx_unlock(&ntoskrnl_dispatchlock);
2860
2861         return (iw);
2862 }
2863
2864 void
2865 IoFreeWorkItem(iw)
2866         io_workitem             *iw;
2867 {
2868         uma_zfree(iw_zone, iw);
2869 }
2870
2871 void
2872 IoQueueWorkItem(iw, iw_func, qtype, ctx)
2873         io_workitem             *iw;
2874         io_workitem_func        iw_func;
2875         uint32_t                qtype;
2876         void                    *ctx;
2877 {
2878         kdpc_queue              *kq;
2879         list_entry              *l;
2880         io_workitem             *cur;
2881         uint8_t                 irql;
2882
2883         kq = wq_queues + iw->iw_idx;
2884
2885         KeAcquireSpinLock(&kq->kq_lock, &irql);
2886
2887         /*
2888          * Traverse the list and make sure this workitem hasn't
2889          * already been inserted. Queuing the same workitem
2890          * twice will hose the list but good.
2891          */
2892
2893         l = kq->kq_disp.nle_flink;
2894         while (l != &kq->kq_disp) {
2895                 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2896                 if (cur == iw) {
2897                         /* Already queued -- do nothing. */
2898                         KeReleaseSpinLock(&kq->kq_lock, irql);
2899                         return;
2900                 }
2901                 l = l->nle_flink;
2902         }
2903
2904         iw->iw_func = iw_func;
2905         iw->iw_ctx = ctx;
2906
2907         InsertTailList((&kq->kq_disp), (&iw->iw_listentry));
2908         KeReleaseSpinLock(&kq->kq_lock, irql);
2909
2910         KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2911 }
2912
2913 static void
2914 ntoskrnl_workitem(dobj, arg)
2915         device_object           *dobj;
2916         void                    *arg;
2917 {
2918         io_workitem             *iw;
2919         work_queue_item         *w;
2920         work_item_func          f;
2921
2922         iw = arg;
2923         w = (work_queue_item *)dobj;
2924         f = (work_item_func)w->wqi_func;
2925         uma_zfree(iw_zone, iw);
2926         MSCALL2(f, w, w->wqi_ctx);
2927 }
2928
2929 /*
2930  * The ExQueueWorkItem() API is deprecated in Windows XP. Microsoft
2931  * warns that it's unsafe and to use IoQueueWorkItem() instead. The
2932  * problem with ExQueueWorkItem() is that it can't guard against
2933  * the condition where a driver submits a job to the work queue and
2934  * is then unloaded before the job is able to run. IoQueueWorkItem()
2935  * acquires a reference to the device's device_object via the
2936  * object manager and retains it until after the job has completed,
2937  * which prevents the driver from being unloaded before the job
2938  * runs. (We don't currently support this behavior, though hopefully
2939  * that will change once the object manager API is fleshed out a bit.)
2940  *
2941  * Having said all that, the ExQueueWorkItem() API remains, because
2942  * there are still other parts of Windows that use it, including
2943  * NDIS itself: NdisScheduleWorkItem() calls ExQueueWorkItem().
2944  * We fake up the ExQueueWorkItem() API on top of our implementation
2945  * of IoQueueWorkItem(). Workitem thread #3 is reserved exclusively
2946  * for ExQueueWorkItem() jobs, and we pass a pointer to the work
2947  * queue item (provided by the caller) in to IoAllocateWorkItem()
2948  * instead of the device_object. We need to save this pointer so
2949  * we can apply a sanity check: as with the DPC queue and other
2950  * workitem queues, we can't allow the same work queue item to
2951  * be queued twice. If it's already pending, we silently return
2952  */
2953
2954 void
2955 ExQueueWorkItem(w, qtype)
2956         work_queue_item         *w;
2957         uint32_t                qtype;
2958 {
2959         io_workitem             *iw;
2960         io_workitem_func        iwf;
2961         kdpc_queue              *kq;
2962         list_entry              *l;
2963         io_workitem             *cur;
2964         uint8_t                 irql;
2965
2966         /*
2967          * We need to do a special sanity test to make sure
2968          * the ExQueueWorkItem() API isn't used to queue
2969          * the same workitem twice. Rather than checking the
2970          * io_workitem pointer itself, we test the attached
2971          * device object, which is really a pointer to the
2972          * legacy work queue item structure.
2973          */
2974
2975         kq = wq_queues + WORKITEM_LEGACY_THREAD;
2976         KeAcquireSpinLock(&kq->kq_lock, &irql);
2977         l = kq->kq_disp.nle_flink;
2978         while (l != &kq->kq_disp) {
2979                 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2980                 if (cur->iw_dobj == (device_object *)w) {
2981                         /* Already queued -- do nothing. */
2982                         KeReleaseSpinLock(&kq->kq_lock, irql);
2983                         return;
2984                 }
2985                 l = l->nle_flink;
2986         }
2987         KeReleaseSpinLock(&kq->kq_lock, irql);
2988
2989         iw = IoAllocateWorkItem((device_object *)w);
2990         if (iw == NULL)
2991                 return;
2992
2993         iw->iw_idx = WORKITEM_LEGACY_THREAD;
2994         iwf = (io_workitem_func)ntoskrnl_findwrap((funcptr)ntoskrnl_workitem);
2995         IoQueueWorkItem(iw, iwf, qtype, iw);
2996 }
2997
2998 static void
2999 RtlZeroMemory(dst, len)
3000         void                    *dst;
3001         size_t                  len;
3002 {
3003         bzero(dst, len);
3004 }
3005
3006 static void
3007 RtlSecureZeroMemory(dst, len)
3008         void                    *dst;
3009         size_t                  len;
3010 {
3011         memset(dst, 0, len);
3012 }
3013
3014 static void
3015 RtlFillMemory(void *dst, size_t len, uint8_t c)
3016 {
3017         memset(dst, c, len);
3018 }
3019
3020 static void
3021 RtlMoveMemory(dst, src, len)
3022         void                    *dst;
3023         const void              *src;
3024         size_t                  len;
3025 {
3026         memmove(dst, src, len);
3027 }
3028
3029 static void
3030 RtlCopyMemory(dst, src, len)
3031         void                    *dst;
3032         const void              *src;
3033         size_t                  len;
3034 {
3035         bcopy(src, dst, len);
3036 }
3037
3038 static size_t
3039 RtlCompareMemory(s1, s2, len)
3040         const void              *s1;
3041         const void              *s2;
3042         size_t                  len;
3043 {
3044         size_t                  i;
3045         uint8_t                 *m1, *m2;
3046
3047         m1 = __DECONST(char *, s1);
3048         m2 = __DECONST(char *, s2);
3049
3050         for (i = 0; i < len && m1[i] == m2[i]; i++);
3051         return (i);
3052 }
3053
3054 void
3055 RtlInitAnsiString(dst, src)
3056         ansi_string             *dst;
3057         char                    *src;
3058 {
3059         ansi_string             *a;
3060
3061         a = dst;
3062         if (a == NULL)
3063                 return;
3064         if (src == NULL) {
3065                 a->as_len = a->as_maxlen = 0;
3066                 a->as_buf = NULL;
3067         } else {
3068                 a->as_buf = src;
3069                 a->as_len = a->as_maxlen = strlen(src);
3070         }
3071 }
3072
3073 void
3074 RtlInitUnicodeString(dst, src)
3075         unicode_string          *dst;
3076         uint16_t                *src;
3077 {
3078         unicode_string          *u;
3079         int                     i;
3080
3081         u = dst;
3082         if (u == NULL)
3083                 return;
3084         if (src == NULL) {
3085                 u->us_len = u->us_maxlen = 0;
3086                 u->us_buf = NULL;
3087         } else {
3088                 i = 0;
3089                 while(src[i] != 0)
3090                         i++;
3091                 u->us_buf = src;
3092                 u->us_len = u->us_maxlen = i * 2;
3093         }
3094 }
3095
3096 ndis_status
3097 RtlUnicodeStringToInteger(ustr, base, val)
3098         unicode_string          *ustr;
3099         uint32_t                base;
3100         uint32_t                *val;
3101 {
3102         uint16_t                *uchr;
3103         int                     len, neg = 0;
3104         char                    abuf[64];
3105         char                    *astr;
3106
3107         uchr = ustr->us_buf;
3108         len = ustr->us_len;
3109         bzero(abuf, sizeof(abuf));
3110
3111         if ((char)((*uchr) & 0xFF) == '-') {
3112                 neg = 1;
3113                 uchr++;
3114                 len -= 2;
3115         } else if ((char)((*uchr) & 0xFF) == '+') {
3116                 neg = 0;
3117                 uchr++;
3118                 len -= 2;
3119         }
3120
3121         if (base == 0) {
3122                 if ((char)((*uchr) & 0xFF) == 'b') {
3123                         base = 2;
3124                         uchr++;
3125                         len -= 2;
3126                 } else if ((char)((*uchr) & 0xFF) == 'o') {
3127                         base = 8;
3128                         uchr++;
3129                         len -= 2;
3130                 } else if ((char)((*uchr) & 0xFF) == 'x') {
3131                         base = 16;
3132                         uchr++;
3133                         len -= 2;
3134                 } else
3135                         base = 10;
3136         }
3137
3138         astr = abuf;
3139         if (neg) {
3140                 strcpy(astr, "-");
3141                 astr++;
3142         }
3143
3144         ntoskrnl_unicode_to_ascii(uchr, astr, len);
3145         *val = strtoul(abuf, NULL, base);
3146
3147         return (STATUS_SUCCESS);
3148 }
3149
3150 void
3151 RtlFreeUnicodeString(ustr)
3152         unicode_string          *ustr;
3153 {
3154         if (ustr->us_buf == NULL)
3155                 return;
3156         ExFreePool(ustr->us_buf);
3157         ustr->us_buf = NULL;
3158 }
3159
3160 void
3161 RtlFreeAnsiString(astr)
3162         ansi_string             *astr;
3163 {
3164         if (astr->as_buf == NULL)
3165                 return;
3166         ExFreePool(astr->as_buf);
3167         astr->as_buf = NULL;
3168 }
3169
3170 static int
3171 atoi(str)
3172         const char              *str;
3173 {
3174         return (int)strtol(str, (char **)NULL, 10);
3175 }
3176
3177 static long
3178 atol(str)
3179         const char              *str;
3180 {
3181         return strtol(str, (char **)NULL, 10);
3182 }
3183
3184 static int
3185 rand(void)
3186 {
3187
3188         return (random());
3189 }
3190
3191 static void
3192 srand(unsigned int seed __unused)
3193 {
3194 }
3195
3196 static uint8_t
3197 IoIsWdmVersionAvailable(uint8_t major, uint8_t minor)
3198 {
3199         if (major == WDM_MAJOR && minor == WDM_MINOR_WINXP)
3200                 return (TRUE);
3201         return (FALSE);
3202 }
3203
3204 static int32_t
3205 IoOpenDeviceRegistryKey(struct device_object *devobj, uint32_t type,
3206     uint32_t mask, void **key)
3207 {
3208         return (NDIS_STATUS_INVALID_DEVICE_REQUEST);
3209 }
3210
3211 static ndis_status
3212 IoGetDeviceObjectPointer(name, reqaccess, fileobj, devobj)
3213         unicode_string          *name;
3214         uint32_t                reqaccess;
3215         void                    *fileobj;
3216         device_object           *devobj;
3217 {
3218         return (STATUS_SUCCESS);
3219 }
3220
3221 static ndis_status
3222 IoGetDeviceProperty(devobj, regprop, buflen, prop, reslen)
3223         device_object           *devobj;
3224         uint32_t                regprop;
3225         uint32_t                buflen;
3226         void                    *prop;
3227         uint32_t                *reslen;
3228 {
3229         driver_object           *drv;
3230         uint16_t                **name;
3231
3232         drv = devobj->do_drvobj;
3233
3234         switch (regprop) {
3235         case DEVPROP_DRIVER_KEYNAME:
3236                 name = prop;
3237                 *name = drv->dro_drivername.us_buf;
3238                 *reslen = drv->dro_drivername.us_len;
3239                 break;
3240         default:
3241                 return (STATUS_INVALID_PARAMETER_2);
3242                 break;
3243         }
3244
3245         return (STATUS_SUCCESS);
3246 }
3247
3248 static void
3249 KeInitializeMutex(kmutex, level)
3250         kmutant                 *kmutex;
3251         uint32_t                level;
3252 {
3253         InitializeListHead((&kmutex->km_header.dh_waitlisthead));
3254         kmutex->km_abandoned = FALSE;
3255         kmutex->km_apcdisable = 1;
3256         kmutex->km_header.dh_sigstate = 1;
3257         kmutex->km_header.dh_type = DISP_TYPE_MUTANT;
3258         kmutex->km_header.dh_size = sizeof(kmutant) / sizeof(uint32_t);
3259         kmutex->km_ownerthread = NULL;
3260 }
3261
3262 static uint32_t
3263 KeReleaseMutex(kmutant *kmutex, uint8_t kwait)
3264 {
3265         uint32_t                prevstate;
3266
3267         mtx_lock(&ntoskrnl_dispatchlock);
3268         prevstate = kmutex->km_header.dh_sigstate;
3269         if (kmutex->km_ownerthread != curthread) {
3270                 mtx_unlock(&ntoskrnl_dispatchlock);
3271                 return (STATUS_MUTANT_NOT_OWNED);
3272         }
3273
3274         kmutex->km_header.dh_sigstate++;
3275         kmutex->km_abandoned = FALSE;
3276
3277         if (kmutex->km_header.dh_sigstate == 1) {
3278                 kmutex->km_ownerthread = NULL;
3279                 ntoskrnl_waittest(&kmutex->km_header, IO_NO_INCREMENT);
3280         }
3281
3282         mtx_unlock(&ntoskrnl_dispatchlock);
3283
3284         return (prevstate);
3285 }
3286
3287 static uint32_t
3288 KeReadStateMutex(kmutex)
3289         kmutant                 *kmutex;
3290 {
3291         return (kmutex->km_header.dh_sigstate);
3292 }
3293
3294 void
3295 KeInitializeEvent(nt_kevent *kevent, uint32_t type, uint8_t state)
3296 {
3297         InitializeListHead((&kevent->k_header.dh_waitlisthead));
3298         kevent->k_header.dh_sigstate = state;
3299         if (type == EVENT_TYPE_NOTIFY)
3300                 kevent->k_header.dh_type = DISP_TYPE_NOTIFICATION_EVENT;
3301         else
3302                 kevent->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_EVENT;
3303         kevent->k_header.dh_size = sizeof(nt_kevent) / sizeof(uint32_t);
3304 }
3305
3306 uint32_t
3307 KeResetEvent(kevent)
3308         nt_kevent               *kevent;
3309 {
3310         uint32_t                prevstate;
3311
3312         mtx_lock(&ntoskrnl_dispatchlock);
3313         prevstate = kevent->k_header.dh_sigstate;
3314         kevent->k_header.dh_sigstate = FALSE;
3315         mtx_unlock(&ntoskrnl_dispatchlock);
3316
3317         return (prevstate);
3318 }
3319
3320 uint32_t
3321 KeSetEvent(nt_kevent *kevent, uint32_t increment, uint8_t kwait)
3322 {
3323         uint32_t                prevstate;
3324         wait_block              *w;
3325         nt_dispatch_header      *dh;
3326         struct thread           *td;
3327         wb_ext                  *we;
3328
3329         mtx_lock(&ntoskrnl_dispatchlock);
3330         prevstate = kevent->k_header.dh_sigstate;
3331         dh = &kevent->k_header;
3332
3333         if (IsListEmpty(&dh->dh_waitlisthead))
3334                 /*
3335                  * If there's nobody in the waitlist, just set
3336                  * the state to signalled.
3337                  */
3338                 dh->dh_sigstate = 1;
3339         else {
3340                 /*
3341                  * Get the first waiter. If this is a synchronization
3342                  * event, just wake up that one thread (don't bother
3343                  * setting the state to signalled since we're supposed
3344                  * to automatically clear synchronization events anyway).
3345                  *
3346                  * If it's a notification event, or the first
3347                  * waiter is doing a WAITTYPE_ALL wait, go through
3348                  * the full wait satisfaction process.
3349                  */
3350                 w = CONTAINING_RECORD(dh->dh_waitlisthead.nle_flink,
3351                     wait_block, wb_waitlist);
3352                 we = w->wb_ext;
3353                 td = we->we_td;
3354                 if (kevent->k_header.dh_type == DISP_TYPE_NOTIFICATION_EVENT ||
3355                     w->wb_waittype == WAITTYPE_ALL) {
3356                         if (prevstate == 0) {
3357                                 dh->dh_sigstate = 1;
3358                                 ntoskrnl_waittest(dh, increment);
3359                         }
3360                 } else {
3361                         w->wb_awakened |= TRUE;
3362                         cv_broadcastpri(&we->we_cv,
3363                             (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
3364                             w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
3365                 }
3366         }
3367
3368         mtx_unlock(&ntoskrnl_dispatchlock);
3369
3370         return (prevstate);
3371 }
3372
3373 void
3374 KeClearEvent(kevent)
3375         nt_kevent               *kevent;
3376 {
3377         kevent->k_header.dh_sigstate = FALSE;
3378 }
3379
3380 uint32_t
3381 KeReadStateEvent(kevent)
3382         nt_kevent               *kevent;
3383 {
3384         return (kevent->k_header.dh_sigstate);
3385 }
3386
3387 /*
3388  * The object manager in Windows is responsible for managing
3389  * references and access to various types of objects, including
3390  * device_objects, events, threads, timers and so on. However,
3391  * there's a difference in the way objects are handled in user
3392  * mode versus kernel mode.
3393  *
3394  * In user mode (i.e. Win32 applications), all objects are
3395  * managed by the object manager. For example, when you create
3396  * a timer or event object, you actually end up with an
3397  * object_header (for the object manager's bookkeeping
3398  * purposes) and an object body (which contains the actual object
3399  * structure, e.g. ktimer, kevent, etc...). This allows Windows
3400  * to manage resource quotas and to enforce access restrictions
3401  * on basically every kind of system object handled by the kernel.
3402  *
3403  * However, in kernel mode, you only end up using the object
3404  * manager some of the time. For example, in a driver, you create
3405  * a timer object by simply allocating the memory for a ktimer
3406  * structure and initializing it with KeInitializeTimer(). Hence,
3407  * the timer has no object_header and no reference counting or
3408  * security/resource checks are done on it. The assumption in
3409  * this case is that if you're running in kernel mode, you know
3410  * what you're doing, and you're already at an elevated privilege
3411  * anyway.
3412  *
3413  * There are some exceptions to this. The two most important ones
3414  * for our purposes are device_objects and threads. We need to use
3415  * the object manager to do reference counting on device_objects,
3416  * and for threads, you can only get a pointer to a thread's
3417  * dispatch header by using ObReferenceObjectByHandle() on the
3418  * handle returned by PsCreateSystemThread().
3419  */
3420
3421 static ndis_status
3422 ObReferenceObjectByHandle(ndis_handle handle, uint32_t reqaccess, void *otype,
3423         uint8_t accessmode, void **object, void **handleinfo)
3424 {
3425         nt_objref               *nr;
3426
3427         nr = malloc(sizeof(nt_objref), M_DEVBUF, M_NOWAIT|M_ZERO);
3428         if (nr == NULL)
3429                 return (STATUS_INSUFFICIENT_RESOURCES);
3430
3431         InitializeListHead((&nr->no_dh.dh_waitlisthead));
3432         nr->no_obj = handle;
3433         nr->no_dh.dh_type = DISP_TYPE_THREAD;
3434         nr->no_dh.dh_sigstate = 0;
3435         nr->no_dh.dh_size = (uint8_t)(sizeof(struct thread) /
3436             sizeof(uint32_t));
3437         TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link);
3438         *object = nr;
3439
3440         return (STATUS_SUCCESS);
3441 }
3442
3443 static void
3444 ObfDereferenceObject(object)
3445         void                    *object;
3446 {
3447         nt_objref               *nr;
3448
3449         nr = object;
3450         TAILQ_REMOVE(&ntoskrnl_reflist, nr, link);
3451         free(nr, M_DEVBUF);
3452 }
3453
3454 static uint32_t
3455 ZwClose(handle)
3456         ndis_handle             handle;
3457 {
3458         return (STATUS_SUCCESS);
3459 }
3460
3461 static uint32_t
3462 WmiQueryTraceInformation(traceclass, traceinfo, infolen, reqlen, buf)
3463         uint32_t                traceclass;
3464         void                    *traceinfo;
3465         uint32_t                infolen;
3466         uint32_t                reqlen;
3467         void                    *buf;
3468 {
3469         return (STATUS_NOT_FOUND);
3470 }
3471
3472 static uint32_t
3473 WmiTraceMessage(uint64_t loghandle, uint32_t messageflags,
3474         void *guid, uint16_t messagenum, ...)
3475 {
3476         return (STATUS_SUCCESS);
3477 }
3478
3479 static uint32_t
3480 IoWMIRegistrationControl(dobj, action)
3481         device_object           *dobj;
3482         uint32_t                action;
3483 {
3484         return (STATUS_SUCCESS);
3485 }
3486
3487 /*
3488  * This is here just in case the thread returns without calling
3489  * PsTerminateSystemThread().
3490  */
3491 static void
3492 ntoskrnl_thrfunc(arg)
3493         void                    *arg;
3494 {
3495         thread_context          *thrctx;
3496         uint32_t (*tfunc)(void *);
3497         void                    *tctx;
3498         uint32_t                rval;
3499
3500         thrctx = arg;
3501         tfunc = thrctx->tc_thrfunc;
3502         tctx = thrctx->tc_thrctx;
3503         free(thrctx, M_TEMP);
3504
3505         rval = MSCALL1(tfunc, tctx);
3506
3507         PsTerminateSystemThread(rval);
3508         return; /* notreached */
3509 }
3510
3511 static ndis_status
3512 PsCreateSystemThread(handle, reqaccess, objattrs, phandle,
3513         clientid, thrfunc, thrctx)
3514         ndis_handle             *handle;
3515         uint32_t                reqaccess;
3516         void                    *objattrs;
3517         ndis_handle             phandle;
3518         void                    *clientid;
3519         void                    *thrfunc;
3520         void                    *thrctx;
3521 {
3522         int                     error;
3523         thread_context          *tc;
3524         struct proc             *p;
3525
3526         tc = malloc(sizeof(thread_context), M_TEMP, M_NOWAIT);
3527         if (tc == NULL)
3528                 return (STATUS_INSUFFICIENT_RESOURCES);
3529
3530         tc->tc_thrctx = thrctx;
3531         tc->tc_thrfunc = thrfunc;
3532
3533         error = kproc_create(ntoskrnl_thrfunc, tc, &p,
3534             RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Kthread %d", ntoskrnl_kth);
3535
3536         if (error) {
3537                 free(tc, M_TEMP);
3538                 return (STATUS_INSUFFICIENT_RESOURCES);
3539         }
3540
3541         *handle = p;
3542         ntoskrnl_kth++;
3543
3544         return (STATUS_SUCCESS);
3545 }
3546
3547 /*
3548  * In Windows, the exit of a thread is an event that you're allowed
3549  * to wait on, assuming you've obtained a reference to the thread using
3550  * ObReferenceObjectByHandle(). Unfortunately, the only way we can
3551  * simulate this behavior is to register each thread we create in a
3552  * reference list, and if someone holds a reference to us, we poke
3553  * them.
3554  */
3555 static ndis_status
3556 PsTerminateSystemThread(status)
3557         ndis_status             status;
3558 {
3559         struct nt_objref        *nr;
3560
3561         mtx_lock(&ntoskrnl_dispatchlock);
3562         TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) {
3563                 if (nr->no_obj != curthread->td_proc)
3564                         continue;
3565                 nr->no_dh.dh_sigstate = 1;
3566                 ntoskrnl_waittest(&nr->no_dh, IO_NO_INCREMENT);
3567                 break;
3568         }
3569         mtx_unlock(&ntoskrnl_dispatchlock);
3570
3571         ntoskrnl_kth--;
3572
3573         kproc_exit(0);
3574         return (0);     /* notreached */
3575 }
3576
3577 static uint32_t
3578 DbgPrint(char *fmt, ...)
3579 {
3580         va_list                 ap;
3581
3582         if (bootverbose) {
3583                 va_start(ap, fmt);
3584                 vprintf(fmt, ap);
3585                 va_end(ap);
3586         }
3587
3588         return (STATUS_SUCCESS);
3589 }
3590
3591 static void
3592 DbgBreakPoint(void)
3593 {
3594
3595         kdb_enter(KDB_WHY_NDIS, "DbgBreakPoint(): breakpoint");
3596 }
3597
3598 static void
3599 KeBugCheckEx(code, param1, param2, param3, param4)
3600     uint32_t                    code;
3601     u_long                      param1;
3602     u_long                      param2;
3603     u_long                      param3;
3604     u_long                      param4;
3605 {
3606         panic("KeBugCheckEx: STOP 0x%X", code);
3607 }
3608
3609 static void
3610 ntoskrnl_timercall(arg)
3611         void                    *arg;
3612 {
3613         ktimer                  *timer;
3614         struct timeval          tv;
3615         kdpc                    *dpc;
3616
3617         mtx_lock(&ntoskrnl_dispatchlock);
3618
3619         timer = arg;
3620
3621 #ifdef NTOSKRNL_DEBUG_TIMERS
3622         ntoskrnl_timer_fires++;
3623 #endif
3624         ntoskrnl_remove_timer(timer);
3625
3626         /*
3627          * This should never happen, but complain
3628          * if it does.
3629          */
3630
3631         if (timer->k_header.dh_inserted == FALSE) {
3632                 mtx_unlock(&ntoskrnl_dispatchlock);
3633                 printf("NTOS: timer %p fired even though "
3634                     "it was canceled\n", timer);
3635                 return;
3636         }
3637
3638         /* Mark the timer as no longer being on the timer queue. */
3639
3640         timer->k_header.dh_inserted = FALSE;
3641
3642         /* Now signal the object and satisfy any waits on it. */
3643
3644         timer->k_header.dh_sigstate = 1;
3645         ntoskrnl_waittest(&timer->k_header, IO_NO_INCREMENT);
3646
3647         /*
3648          * If this is a periodic timer, re-arm it
3649          * so it will fire again. We do this before
3650          * calling any deferred procedure calls because
3651          * it's possible the DPC might cancel the timer,
3652          * in which case it would be wrong for us to
3653          * re-arm it again afterwards.
3654          */
3655
3656         if (timer->k_period) {
3657                 tv.tv_sec = 0;
3658                 tv.tv_usec = timer->k_period * 1000;
3659                 timer->k_header.dh_inserted = TRUE;
3660                 ntoskrnl_insert_timer(timer, tvtohz(&tv));
3661 #ifdef NTOSKRNL_DEBUG_TIMERS
3662                 ntoskrnl_timer_reloads++;
3663 #endif
3664         }
3665
3666         dpc = timer->k_dpc;
3667
3668         mtx_unlock(&ntoskrnl_dispatchlock);
3669
3670         /* If there's a DPC associated with the timer, queue it up. */
3671
3672         if (dpc != NULL)
3673                 KeInsertQueueDpc(dpc, NULL, NULL);
3674 }
3675
3676 #ifdef NTOSKRNL_DEBUG_TIMERS
3677 static int
3678 sysctl_show_timers(SYSCTL_HANDLER_ARGS)
3679 {
3680         int                     ret;
3681
3682         ret = 0;
3683         ntoskrnl_show_timers();
3684         return (sysctl_handle_int(oidp, &ret, 0, req));
3685 }
3686
3687 static void
3688 ntoskrnl_show_timers()
3689 {
3690         int                     i = 0;
3691         list_entry              *l;
3692
3693         mtx_lock_spin(&ntoskrnl_calllock);
3694         l = ntoskrnl_calllist.nle_flink;
3695         while(l != &ntoskrnl_calllist) {
3696                 i++;
3697                 l = l->nle_flink;
3698         }
3699         mtx_unlock_spin(&ntoskrnl_calllock);
3700
3701         printf("\n");
3702         printf("%d timers available (out of %d)\n", i, NTOSKRNL_TIMEOUTS);
3703         printf("timer sets: %qu\n", ntoskrnl_timer_sets);
3704         printf("timer reloads: %qu\n", ntoskrnl_timer_reloads);
3705         printf("timer cancels: %qu\n", ntoskrnl_timer_cancels);
3706         printf("timer fires: %qu\n", ntoskrnl_timer_fires);
3707         printf("\n");
3708 }
3709 #endif
3710
3711 /*
3712  * Must be called with dispatcher lock held.
3713  */
3714
3715 static void
3716 ntoskrnl_insert_timer(timer, ticks)
3717         ktimer                  *timer;
3718         int                     ticks;
3719 {
3720         callout_entry           *e;
3721         list_entry              *l;
3722         struct callout          *c;
3723
3724         /*
3725          * Try and allocate a timer.
3726          */
3727         mtx_lock_spin(&ntoskrnl_calllock);
3728         if (IsListEmpty(&ntoskrnl_calllist)) {
3729                 mtx_unlock_spin(&ntoskrnl_calllock);
3730 #ifdef NTOSKRNL_DEBUG_TIMERS
3731                 ntoskrnl_show_timers();
3732 #endif
3733                 panic("out of timers!");
3734         }
3735         l = RemoveHeadList(&ntoskrnl_calllist);
3736         mtx_unlock_spin(&ntoskrnl_calllock);
3737
3738         e = CONTAINING_RECORD(l, callout_entry, ce_list);
3739         c = &e->ce_callout;
3740
3741         timer->k_callout = c;
3742
3743         callout_init(c, 1);
3744         callout_reset(c, ticks, ntoskrnl_timercall, timer);
3745 }
3746
3747 static void
3748 ntoskrnl_remove_timer(timer)
3749         ktimer                  *timer;
3750 {
3751         callout_entry           *e;
3752
3753         e = (callout_entry *)timer->k_callout;
3754         callout_stop(timer->k_callout);
3755
3756         mtx_lock_spin(&ntoskrnl_calllock);
3757         InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
3758         mtx_unlock_spin(&ntoskrnl_calllock);
3759 }
3760
3761 void
3762 KeInitializeTimer(timer)
3763         ktimer                  *timer;
3764 {
3765         if (timer == NULL)
3766                 return;
3767
3768         KeInitializeTimerEx(timer,  EVENT_TYPE_NOTIFY);
3769 }
3770
3771 void
3772 KeInitializeTimerEx(timer, type)
3773         ktimer                  *timer;
3774         uint32_t                type;
3775 {
3776         if (timer == NULL)
3777                 return;
3778
3779         bzero((char *)timer, sizeof(ktimer));
3780         InitializeListHead((&timer->k_header.dh_waitlisthead));
3781         timer->k_header.dh_sigstate = FALSE;
3782         timer->k_header.dh_inserted = FALSE;
3783         if (type == EVENT_TYPE_NOTIFY)
3784                 timer->k_header.dh_type = DISP_TYPE_NOTIFICATION_TIMER;
3785         else
3786                 timer->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_TIMER;
3787         timer->k_header.dh_size = sizeof(ktimer) / sizeof(uint32_t);
3788 }
3789
3790 /*
3791  * DPC subsystem. A Windows Defered Procedure Call has the following
3792  * properties:
3793  * - It runs at DISPATCH_LEVEL.
3794  * - It can have one of 3 importance values that control when it
3795  *   runs relative to other DPCs in the queue.
3796  * - On SMP systems, it can be set to run on a specific processor.
3797  * In order to satisfy the last property, we create a DPC thread for
3798  * each CPU in the system and bind it to that CPU. Each thread
3799  * maintains three queues with different importance levels, which
3800  * will be processed in order from lowest to highest.
3801  *
3802  * In Windows, interrupt handlers run as DPCs. (Not to be confused
3803  * with ISRs, which run in interrupt context and can preempt DPCs.)
3804  * ISRs are given the highest importance so that they'll take
3805  * precedence over timers and other things.
3806  */
3807
3808 static void
3809 ntoskrnl_dpc_thread(arg)
3810         void                    *arg;
3811 {
3812         kdpc_queue              *kq;
3813         kdpc                    *d;
3814         list_entry              *l;
3815         uint8_t                 irql;
3816
3817         kq = arg;
3818
3819         InitializeListHead(&kq->kq_disp);
3820         kq->kq_td = curthread;
3821         kq->kq_exit = 0;
3822         kq->kq_running = FALSE;
3823         KeInitializeSpinLock(&kq->kq_lock);
3824         KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
3825         KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE);
3826
3827         /*
3828          * Elevate our priority. DPCs are used to run interrupt
3829          * handlers, and they should trigger as soon as possible
3830          * once scheduled by an ISR.
3831          */
3832
3833         thread_lock(curthread);
3834 #ifdef NTOSKRNL_MULTIPLE_DPCS
3835         sched_bind(curthread, kq->kq_cpu);
3836 #endif
3837         sched_prio(curthread, PRI_MIN_KERN);
3838         thread_unlock(curthread);
3839
3840         while (1) {
3841                 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
3842
3843                 KeAcquireSpinLock(&kq->kq_lock, &irql);
3844
3845                 if (kq->kq_exit) {
3846                         kq->kq_exit = 0;
3847                         KeReleaseSpinLock(&kq->kq_lock, irql);
3848                         break;
3849                 }
3850
3851                 kq->kq_running = TRUE;
3852
3853                 while (!IsListEmpty(&kq->kq_disp)) {
3854                         l = RemoveHeadList((&kq->kq_disp));
3855                         d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3856                         InitializeListHead((&d->k_dpclistentry));
3857                         KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
3858                         MSCALL4(d->k_deferedfunc, d, d->k_deferredctx,
3859                             d->k_sysarg1, d->k_sysarg2);
3860                         KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3861                 }
3862
3863                 kq->kq_running = FALSE;
3864
3865                 KeReleaseSpinLock(&kq->kq_lock, irql);
3866
3867                 KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE);
3868         }
3869
3870         kproc_exit(0);
3871         return; /* notreached */
3872 }
3873
3874 static void
3875 ntoskrnl_destroy_dpc_threads(void)
3876 {
3877         kdpc_queue              *kq;
3878         kdpc                    dpc;
3879         int                     i;
3880
3881         kq = kq_queues;
3882 #ifdef NTOSKRNL_MULTIPLE_DPCS
3883         for (i = 0; i < mp_ncpus; i++) {
3884 #else
3885         for (i = 0; i < 1; i++) {
3886 #endif
3887                 kq += i;
3888
3889                 kq->kq_exit = 1;
3890                 KeInitializeDpc(&dpc, NULL, NULL);
3891                 KeSetTargetProcessorDpc(&dpc, i);
3892                 KeInsertQueueDpc(&dpc, NULL, NULL);
3893                 while (kq->kq_exit)
3894                         tsleep(kq->kq_td->td_proc, PWAIT, "dpcw", hz/10);
3895         }
3896 }
3897
3898 static uint8_t
3899 ntoskrnl_insert_dpc(head, dpc)
3900         list_entry              *head;
3901         kdpc                    *dpc;
3902 {
3903         list_entry              *l;
3904         kdpc                    *d;
3905
3906         l = head->nle_flink;
3907         while (l != head) {
3908                 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3909                 if (d == dpc)
3910                         return (FALSE);
3911                 l = l->nle_flink;
3912         }
3913
3914         if (dpc->k_importance == KDPC_IMPORTANCE_LOW)
3915                 InsertTailList((head), (&dpc->k_dpclistentry));
3916         else
3917                 InsertHeadList((head), (&dpc->k_dpclistentry));
3918
3919         return (TRUE);
3920 }
3921
3922 void
3923 KeInitializeDpc(dpc, dpcfunc, dpcctx)
3924         kdpc                    *dpc;
3925         void                    *dpcfunc;
3926         void                    *dpcctx;
3927 {
3928
3929         if (dpc == NULL)
3930                 return;
3931
3932         dpc->k_deferedfunc = dpcfunc;
3933         dpc->k_deferredctx = dpcctx;
3934         dpc->k_num = KDPC_CPU_DEFAULT;
3935         dpc->k_importance = KDPC_IMPORTANCE_MEDIUM;
3936         InitializeListHead((&dpc->k_dpclistentry));
3937 }
3938
3939 uint8_t
3940 KeInsertQueueDpc(dpc, sysarg1, sysarg2)
3941         kdpc                    *dpc;
3942         void                    *sysarg1;
3943         void                    *sysarg2;
3944 {
3945         kdpc_queue              *kq;
3946         uint8_t                 r;
3947         uint8_t                 irql;
3948
3949         if (dpc == NULL)
3950                 return (FALSE);
3951
3952         kq = kq_queues;
3953
3954 #ifdef NTOSKRNL_MULTIPLE_DPCS
3955         KeRaiseIrql(DISPATCH_LEVEL, &irql);
3956
3957         /*
3958          * By default, the DPC is queued to run on the same CPU
3959          * that scheduled it.
3960          */
3961
3962         if (dpc->k_num == KDPC_CPU_DEFAULT)
3963                 kq += curthread->td_oncpu;
3964         else
3965                 kq += dpc->k_num;
3966         KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3967 #else
3968         KeAcquireSpinLock(&kq->kq_lock, &irql);
3969 #endif
3970
3971         r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc);
3972         if (r == TRUE) {
3973                 dpc->k_sysarg1 = sysarg1;
3974                 dpc->k_sysarg2 = sysarg2;
3975         }
3976         KeReleaseSpinLock(&kq->kq_lock, irql);
3977
3978         if (r == FALSE)
3979                 return (r);
3980
3981         KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
3982
3983         return (r);
3984 }
3985
3986 uint8_t
3987 KeRemoveQueueDpc(dpc)
3988         kdpc                    *dpc;
3989 {
3990         kdpc_queue              *kq;
3991         uint8_t                 irql;
3992
3993         if (dpc == NULL)
3994                 return (FALSE);
3995
3996 #ifdef NTOSKRNL_MULTIPLE_DPCS
3997         KeRaiseIrql(DISPATCH_LEVEL, &irql);
3998
3999         kq = kq_queues + dpc->k_num;
4000
4001         KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
4002 #else
4003         kq = kq_queues;
4004         KeAcquireSpinLock(&kq->kq_lock, &irql);
4005 #endif
4006
4007         if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) {
4008                 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
4009                 KeLowerIrql(irql);
4010                 return (FALSE);
4011         }
4012
4013         RemoveEntryList((&dpc->k_dpclistentry));
4014         InitializeListHead((&dpc->k_dpclistentry));
4015
4016         KeReleaseSpinLock(&kq->kq_lock, irql);
4017
4018         return (TRUE);
4019 }
4020
4021 void
4022 KeSetImportanceDpc(dpc, imp)
4023         kdpc                    *dpc;
4024         uint32_t                imp;
4025 {
4026         if (imp != KDPC_IMPORTANCE_LOW &&
4027             imp != KDPC_IMPORTANCE_MEDIUM &&
4028             imp != KDPC_IMPORTANCE_HIGH)
4029                 return;
4030
4031         dpc->k_importance = (uint8_t)imp;
4032 }
4033
4034 void
4035 KeSetTargetProcessorDpc(kdpc *dpc, uint8_t cpu)
4036 {
4037         if (cpu > mp_ncpus)
4038                 return;
4039
4040         dpc->k_num = cpu;
4041 }
4042
4043 void
4044 KeFlushQueuedDpcs(void)
4045 {
4046         kdpc_queue              *kq;
4047         int                     i;
4048
4049         /*
4050          * Poke each DPC queue and wait
4051          * for them to drain.
4052          */
4053
4054 #ifdef NTOSKRNL_MULTIPLE_DPCS
4055         for (i = 0; i < mp_ncpus; i++) {
4056 #else
4057         for (i = 0; i < 1; i++) {
4058 #endif
4059                 kq = kq_queues + i;
4060                 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
4061                 KeWaitForSingleObject(&kq->kq_done, 0, 0, TRUE, NULL);
4062         }
4063 }
4064
4065 uint32_t
4066 KeGetCurrentProcessorNumber(void)
4067 {
4068         return ((uint32_t)curthread->td_oncpu);
4069 }
4070
4071 uint8_t
4072 KeSetTimerEx(timer, duetime, period, dpc)
4073         ktimer                  *timer;
4074         int64_t                 duetime;
4075         uint32_t                period;
4076         kdpc                    *dpc;
4077 {
4078         struct timeval          tv;
4079         uint64_t                curtime;
4080         uint8_t                 pending;
4081
4082         if (timer == NULL)
4083                 return (FALSE);
4084
4085         mtx_lock(&ntoskrnl_dispatchlock);
4086
4087         if (timer->k_header.dh_inserted == TRUE) {
4088                 ntoskrnl_remove_timer(timer);
4089 #ifdef NTOSKRNL_DEBUG_TIMERS
4090                 ntoskrnl_timer_cancels++;
4091 #endif
4092                 timer->k_header.dh_inserted = FALSE;
4093                 pending = TRUE;
4094         } else
4095                 pending = FALSE;
4096
4097         timer->k_duetime = duetime;
4098         timer->k_period = period;
4099         timer->k_header.dh_sigstate = FALSE;
4100         timer->k_dpc = dpc;
4101
4102         if (duetime < 0) {
4103                 tv.tv_sec = - (duetime) / 10000000;
4104                 tv.tv_usec = (- (duetime) / 10) -
4105                     (tv.tv_sec * 1000000);
4106         } else {
4107                 ntoskrnl_time(&curtime);
4108                 if (duetime < curtime)
4109                         tv.tv_sec = tv.tv_usec = 0;
4110                 else {
4111                         tv.tv_sec = ((duetime) - curtime) / 10000000;
4112                         tv.tv_usec = ((duetime) - curtime) / 10 -
4113                             (tv.tv_sec * 1000000);
4114                 }
4115         }
4116
4117         timer->k_header.dh_inserted = TRUE;
4118         ntoskrnl_insert_timer(timer, tvtohz(&tv));
4119 #ifdef NTOSKRNL_DEBUG_TIMERS
4120         ntoskrnl_timer_sets++;
4121 #endif
4122
4123         mtx_unlock(&ntoskrnl_dispatchlock);
4124
4125         return (pending);
4126 }
4127
4128 uint8_t
4129 KeSetTimer(timer, duetime, dpc)
4130         ktimer                  *timer;
4131         int64_t                 duetime;
4132         kdpc                    *dpc;
4133 {
4134         return (KeSetTimerEx(timer, duetime, 0, dpc));
4135 }
4136
4137 /*
4138  * The Windows DDK documentation seems to say that cancelling
4139  * a timer that has a DPC will result in the DPC also being
4140  * cancelled, but this isn't really the case.
4141  */
4142
4143 uint8_t
4144 KeCancelTimer(timer)
4145         ktimer                  *timer;
4146 {
4147         uint8_t                 pending;
4148
4149         if (timer == NULL)
4150                 return (FALSE);
4151
4152         mtx_lock(&ntoskrnl_dispatchlock);
4153
4154         pending = timer->k_header.dh_inserted;
4155
4156         if (timer->k_header.dh_inserted == TRUE) {
4157                 timer->k_header.dh_inserted = FALSE;
4158                 ntoskrnl_remove_timer(timer);
4159 #ifdef NTOSKRNL_DEBUG_TIMERS
4160                 ntoskrnl_timer_cancels++;
4161 #endif
4162         }
4163
4164         mtx_unlock(&ntoskrnl_dispatchlock);
4165
4166         return (pending);
4167 }
4168
4169 uint8_t
4170 KeReadStateTimer(timer)
4171         ktimer                  *timer;
4172 {
4173         return (timer->k_header.dh_sigstate);
4174 }
4175
4176 static int32_t
4177 KeDelayExecutionThread(uint8_t wait_mode, uint8_t alertable, int64_t *interval)
4178 {
4179         ktimer                  timer;
4180
4181         if (wait_mode != 0)
4182                 panic("invalid wait_mode %d", wait_mode);
4183
4184         KeInitializeTimer(&timer);
4185         KeSetTimer(&timer, *interval, NULL);
4186         KeWaitForSingleObject(&timer, 0, 0, alertable, NULL);
4187
4188         return STATUS_SUCCESS;
4189 }
4190
4191 static uint64_t
4192 KeQueryInterruptTime(void)
4193 {
4194         int ticks;
4195         struct timeval tv;
4196
4197         getmicrouptime(&tv);
4198
4199         ticks = tvtohz(&tv);
4200
4201         return ticks * howmany(10000000, hz);
4202 }
4203
4204 static struct thread *
4205 KeGetCurrentThread(void)
4206 {
4207
4208         return curthread;
4209 }
4210
4211 static int32_t
4212 KeSetPriorityThread(td, pri)
4213         struct thread   *td;
4214         int32_t         pri;
4215 {
4216         int32_t old;
4217
4218         if (td == NULL)
4219                 return LOW_REALTIME_PRIORITY;
4220
4221         if (td->td_priority <= PRI_MIN_KERN)
4222                 old = HIGH_PRIORITY;
4223         else if (td->td_priority >= PRI_MAX_KERN)
4224                 old = LOW_PRIORITY;
4225         else
4226                 old = LOW_REALTIME_PRIORITY;
4227
4228         thread_lock(td);
4229         if (pri == HIGH_PRIORITY)
4230                 sched_prio(td, PRI_MIN_KERN);
4231         if (pri == LOW_REALTIME_PRIORITY)
4232                 sched_prio(td, PRI_MIN_KERN + (PRI_MAX_KERN - PRI_MIN_KERN) / 2);
4233         if (pri == LOW_PRIORITY)
4234                 sched_prio(td, PRI_MAX_KERN);
4235         thread_unlock(td);
4236
4237         return old;
4238 }
4239
4240 static void
4241 dummy()
4242 {
4243         printf("ntoskrnl dummy called...\n");
4244 }
4245
4246 image_patch_table ntoskrnl_functbl[] = {
4247         IMPORT_SFUNC(RtlZeroMemory, 2),
4248         IMPORT_SFUNC(RtlSecureZeroMemory, 2),
4249         IMPORT_SFUNC(RtlFillMemory, 3),
4250         IMPORT_SFUNC(RtlMoveMemory, 3),
4251         IMPORT_SFUNC(RtlCharToInteger, 3),
4252         IMPORT_SFUNC(RtlCopyMemory, 3),
4253         IMPORT_SFUNC(RtlCopyString, 2),
4254         IMPORT_SFUNC(RtlCompareMemory, 3),
4255         IMPORT_SFUNC(RtlEqualUnicodeString, 3),
4256         IMPORT_SFUNC(RtlCopyUnicodeString, 2),
4257         IMPORT_SFUNC(RtlUnicodeStringToAnsiString, 3),
4258         IMPORT_SFUNC(RtlAnsiStringToUnicodeString, 3),
4259         IMPORT_SFUNC(RtlInitAnsiString, 2),
4260         IMPORT_SFUNC_MAP(RtlInitString, RtlInitAnsiString, 2),
4261         IMPORT_SFUNC(RtlInitUnicodeString, 2),
4262         IMPORT_SFUNC(RtlFreeAnsiString, 1),
4263         IMPORT_SFUNC(RtlFreeUnicodeString, 1),
4264         IMPORT_SFUNC(RtlUnicodeStringToInteger, 3),
4265         IMPORT_CFUNC(sprintf, 0),
4266         IMPORT_CFUNC(vsprintf, 0),
4267         IMPORT_CFUNC_MAP(_snprintf, snprintf, 0),
4268         IMPORT_CFUNC_MAP(_vsnprintf, vsnprintf, 0),
4269         IMPORT_CFUNC(DbgPrint, 0),
4270         IMPORT_SFUNC(DbgBreakPoint, 0),
4271         IMPORT_SFUNC(KeBugCheckEx, 5),
4272         IMPORT_CFUNC(strncmp, 0),
4273         IMPORT_CFUNC(strcmp, 0),
4274         IMPORT_CFUNC_MAP(stricmp, strcasecmp, 0),
4275         IMPORT_CFUNC(strncpy, 0),
4276         IMPORT_CFUNC(strcpy, 0),
4277         IMPORT_CFUNC(strlen, 0),
4278         IMPORT_CFUNC_MAP(toupper, ntoskrnl_toupper, 0),
4279         IMPORT_CFUNC_MAP(tolower, ntoskrnl_tolower, 0),
4280         IMPORT_CFUNC_MAP(strstr, ntoskrnl_strstr, 0),
4281         IMPORT_CFUNC_MAP(strncat, ntoskrnl_strncat, 0),
4282         IMPORT_CFUNC_MAP(strchr, index, 0),
4283         IMPORT_CFUNC_MAP(strrchr, rindex, 0),
4284         IMPORT_CFUNC(memcpy, 0),
4285         IMPORT_CFUNC_MAP(memmove, ntoskrnl_memmove, 0),
4286         IMPORT_CFUNC_MAP(memset, ntoskrnl_memset, 0),
4287         IMPORT_CFUNC_MAP(memchr, ntoskrnl_memchr, 0),
4288         IMPORT_SFUNC(IoAllocateDriverObjectExtension, 4),
4289         IMPORT_SFUNC(IoGetDriverObjectExtension, 2),
4290         IMPORT_FFUNC(IofCallDriver, 2),
4291         IMPORT_FFUNC(IofCompleteRequest, 2),
4292         IMPORT_SFUNC(IoAcquireCancelSpinLock, 1),
4293         IMPORT_SFUNC(IoReleaseCancelSpinLock, 1),
4294         IMPORT_SFUNC(IoCancelIrp, 1),
4295         IMPORT_SFUNC(IoConnectInterrupt, 11),
4296         IMPORT_SFUNC(IoDisconnectInterrupt, 1),
4297         IMPORT_SFUNC(IoCreateDevice, 7),
4298         IMPORT_SFUNC(IoDeleteDevice, 1),
4299         IMPORT_SFUNC(IoGetAttachedDevice, 1),
4300         IMPORT_SFUNC(IoAttachDeviceToDeviceStack, 2),
4301         IMPORT_SFUNC(IoDetachDevice, 1),
4302         IMPORT_SFUNC(IoBuildSynchronousFsdRequest, 7),
4303         IMPORT_SFUNC(IoBuildAsynchronousFsdRequest, 6),
4304         IMPORT_SFUNC(IoBuildDeviceIoControlRequest, 9),
4305         IMPORT_SFUNC(IoAllocateIrp, 2),
4306         IMPORT_SFUNC(IoReuseIrp, 2),
4307         IMPORT_SFUNC(IoMakeAssociatedIrp, 2),
4308         IMPORT_SFUNC(IoFreeIrp, 1),
4309         IMPORT_SFUNC(IoInitializeIrp, 3),
4310         IMPORT_SFUNC(KeAcquireInterruptSpinLock, 1),
4311         IMPORT_SFUNC(KeReleaseInterruptSpinLock, 2),
4312         IMPORT_SFUNC(KeSynchronizeExecution, 3),
4313         IMPORT_SFUNC(KeWaitForSingleObject, 5),
4314         IMPORT_SFUNC(KeWaitForMultipleObjects, 8),
4315         IMPORT_SFUNC(_allmul, 4),
4316         IMPORT_SFUNC(_alldiv, 4),
4317         IMPORT_SFUNC(_allrem, 4),
4318         IMPORT_RFUNC(_allshr, 0),
4319         IMPORT_RFUNC(_allshl, 0),
4320         IMPORT_SFUNC(_aullmul, 4),
4321         IMPORT_SFUNC(_aulldiv, 4),
4322         IMPORT_SFUNC(_aullrem, 4),
4323         IMPORT_RFUNC(_aullshr, 0),
4324         IMPORT_RFUNC(_aullshl, 0),
4325         IMPORT_CFUNC(atoi, 0),
4326         IMPORT_CFUNC(atol, 0),
4327         IMPORT_CFUNC(rand, 0),
4328         IMPORT_CFUNC(srand, 0),
4329         IMPORT_SFUNC(WRITE_REGISTER_USHORT, 2),
4330         IMPORT_SFUNC(READ_REGISTER_USHORT, 1),
4331         IMPORT_SFUNC(WRITE_REGISTER_ULONG, 2),
4332         IMPORT_SFUNC(READ_REGISTER_ULONG, 1),
4333         IMPORT_SFUNC(READ_REGISTER_UCHAR, 1),
4334         IMPORT_SFUNC(WRITE_REGISTER_UCHAR, 2),
4335         IMPORT_SFUNC(ExInitializePagedLookasideList, 7),
4336         IMPORT_SFUNC(ExDeletePagedLookasideList, 1),
4337         IMPORT_SFUNC(ExInitializeNPagedLookasideList, 7),
4338         IMPORT_SFUNC(ExDeleteNPagedLookasideList, 1),
4339         IMPORT_FFUNC(InterlockedPopEntrySList, 1),
4340         IMPORT_FFUNC(InitializeSListHead, 1),
4341         IMPORT_FFUNC(InterlockedPushEntrySList, 2),
4342         IMPORT_SFUNC(ExQueryDepthSList, 1),
4343         IMPORT_FFUNC_MAP(ExpInterlockedPopEntrySList,
4344                 InterlockedPopEntrySList, 1),
4345         IMPORT_FFUNC_MAP(ExpInterlockedPushEntrySList,
4346                 InterlockedPushEntrySList, 2),
4347         IMPORT_FFUNC(ExInterlockedPopEntrySList, 2),
4348         IMPORT_FFUNC(ExInterlockedPushEntrySList, 3),
4349         IMPORT_SFUNC(ExAllocatePoolWithTag, 3),
4350         IMPORT_SFUNC(ExFreePoolWithTag, 2),
4351         IMPORT_SFUNC(ExFreePool, 1),
4352 #ifdef __i386__
4353         IMPORT_FFUNC(KefAcquireSpinLockAtDpcLevel, 1),
4354         IMPORT_FFUNC(KefReleaseSpinLockFromDpcLevel,1),
4355         IMPORT_FFUNC(KeAcquireSpinLockRaiseToDpc, 1),
4356 #else
4357         /*
4358          * For AMD64, we can get away with just mapping
4359          * KeAcquireSpinLockRaiseToDpc() directly to KfAcquireSpinLock()
4360          * because the calling conventions end up being the same.
4361          * On i386, we have to be careful because KfAcquireSpinLock()
4362          * is _fastcall but KeAcquireSpinLockRaiseToDpc() isn't.
4363          */
4364         IMPORT_SFUNC(KeAcquireSpinLockAtDpcLevel, 1),
4365         IMPORT_SFUNC(KeReleaseSpinLockFromDpcLevel, 1),
4366         IMPORT_SFUNC_MAP(KeAcquireSpinLockRaiseToDpc, KfAcquireSpinLock, 1),
4367 #endif
4368         IMPORT_SFUNC_MAP(KeReleaseSpinLock, KfReleaseSpinLock, 1),
4369         IMPORT_FFUNC(InterlockedIncrement, 1),
4370         IMPORT_FFUNC(InterlockedDecrement, 1),
4371         IMPORT_FFUNC(InterlockedExchange, 2),
4372         IMPORT_FFUNC(ExInterlockedAddLargeStatistic, 2),
4373         IMPORT_SFUNC(IoAllocateMdl, 5),
4374         IMPORT_SFUNC(IoFreeMdl, 1),
4375         IMPORT_SFUNC(MmAllocateContiguousMemory, 2 + 1),
4376         IMPORT_SFUNC(MmAllocateContiguousMemorySpecifyCache, 5 + 3),
4377         IMPORT_SFUNC(MmFreeContiguousMemory, 1),
4378         IMPORT_SFUNC(MmFreeContiguousMemorySpecifyCache, 3),
4379         IMPORT_SFUNC(MmSizeOfMdl, 1),
4380         IMPORT_SFUNC(MmMapLockedPages, 2),
4381         IMPORT_SFUNC(MmMapLockedPagesSpecifyCache, 6),
4382         IMPORT_SFUNC(MmUnmapLockedPages, 2),
4383         IMPORT_SFUNC(MmBuildMdlForNonPagedPool, 1),
4384         IMPORT_SFUNC(MmGetPhysicalAddress, 1),
4385         IMPORT_SFUNC(MmGetSystemRoutineAddress, 1),
4386         IMPORT_SFUNC(MmIsAddressValid, 1),
4387         IMPORT_SFUNC(MmMapIoSpace, 3 + 1),
4388         IMPORT_SFUNC(MmUnmapIoSpace, 2),
4389         IMPORT_SFUNC(KeInitializeSpinLock, 1),
4390         IMPORT_SFUNC(IoIsWdmVersionAvailable, 2),
4391         IMPORT_SFUNC(IoOpenDeviceRegistryKey, 4),
4392         IMPORT_SFUNC(IoGetDeviceObjectPointer, 4),
4393         IMPORT_SFUNC(IoGetDeviceProperty, 5),
4394         IMPORT_SFUNC(IoAllocateWorkItem, 1),
4395         IMPORT_SFUNC(IoFreeWorkItem, 1),
4396         IMPORT_SFUNC(IoQueueWorkItem, 4),
4397         IMPORT_SFUNC(ExQueueWorkItem, 2),
4398         IMPORT_SFUNC(ntoskrnl_workitem, 2),
4399         IMPORT_SFUNC(KeInitializeMutex, 2),
4400         IMPORT_SFUNC(KeReleaseMutex, 2),
4401         IMPORT_SFUNC(KeReadStateMutex, 1),
4402         IMPORT_SFUNC(KeInitializeEvent, 3),
4403         IMPORT_SFUNC(KeSetEvent, 3),
4404         IMPORT_SFUNC(KeResetEvent, 1),
4405         IMPORT_SFUNC(KeClearEvent, 1),
4406         IMPORT_SFUNC(KeReadStateEvent, 1),
4407         IMPORT_SFUNC(KeInitializeTimer, 1),
4408         IMPORT_SFUNC(KeInitializeTimerEx, 2),
4409         IMPORT_SFUNC(KeSetTimer, 3),
4410         IMPORT_SFUNC(KeSetTimerEx, 4),
4411         IMPORT_SFUNC(KeCancelTimer, 1),
4412         IMPORT_SFUNC(KeReadStateTimer, 1),
4413         IMPORT_SFUNC(KeInitializeDpc, 3),
4414         IMPORT_SFUNC(KeInsertQueueDpc, 3),
4415         IMPORT_SFUNC(KeRemoveQueueDpc, 1),
4416         IMPORT_SFUNC(KeSetImportanceDpc, 2),
4417         IMPORT_SFUNC(KeSetTargetProcessorDpc, 2),
4418         IMPORT_SFUNC(KeFlushQueuedDpcs, 0),
4419         IMPORT_SFUNC(KeGetCurrentProcessorNumber, 1),
4420         IMPORT_SFUNC(ObReferenceObjectByHandle, 6),
4421         IMPORT_FFUNC(ObfDereferenceObject, 1),
4422         IMPORT_SFUNC(ZwClose, 1),
4423         IMPORT_SFUNC(PsCreateSystemThread, 7),
4424         IMPORT_SFUNC(PsTerminateSystemThread, 1),
4425         IMPORT_SFUNC(IoWMIRegistrationControl, 2),
4426         IMPORT_SFUNC(WmiQueryTraceInformation, 5),
4427         IMPORT_CFUNC(WmiTraceMessage, 0),
4428         IMPORT_SFUNC(KeQuerySystemTime, 1),
4429         IMPORT_CFUNC(KeTickCount, 0),
4430         IMPORT_SFUNC(KeDelayExecutionThread, 3),
4431         IMPORT_SFUNC(KeQueryInterruptTime, 0),
4432         IMPORT_SFUNC(KeGetCurrentThread, 0),
4433         IMPORT_SFUNC(KeSetPriorityThread, 2),
4434
4435         /*
4436          * This last entry is a catch-all for any function we haven't
4437          * implemented yet. The PE import list patching routine will
4438          * use it for any function that doesn't have an explicit match
4439          * in this table.
4440          */
4441
4442         { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL },
4443
4444         /* End of list. */
4445         { NULL, NULL, NULL }
4446 };