2 * SPDX-License-Identifier: BSD-4-Clause
5 * Bill Paul <wpaul@windriver.com>. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. All advertising materials mentioning features or use of this software
16 * must display the following acknowledgement:
17 * This product includes software developed by Bill Paul.
18 * 4. Neither the name of the author nor the names of any co-contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32 * THE POSSIBILITY OF SUCH DAMAGE.
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD$");
38 #include <sys/ctype.h>
39 #include <sys/unistd.h>
40 #include <sys/param.h>
41 #include <sys/types.h>
42 #include <sys/errno.h>
43 #include <sys/systm.h>
44 #include <sys/malloc.h>
46 #include <sys/mutex.h>
48 #include <sys/callout.h>
50 #include <sys/kernel.h>
52 #include <sys/condvar.h>
53 #include <sys/kthread.h>
54 #include <sys/module.h>
56 #include <sys/sched.h>
57 #include <sys/sysctl.h>
59 #include <machine/atomic.h>
60 #include <machine/bus.h>
61 #include <machine/stdarg.h>
62 #include <machine/resource.h>
68 #include <vm/vm_param.h>
71 #include <vm/vm_kern.h>
72 #include <vm/vm_map.h>
73 #include <vm/vm_extern.h>
75 #include <compat/ndis/pe_var.h>
76 #include <compat/ndis/cfg_var.h>
77 #include <compat/ndis/resource_var.h>
78 #include <compat/ndis/ntoskrnl_var.h>
79 #include <compat/ndis/hal_var.h>
80 #include <compat/ndis/ndis_var.h>
82 #ifdef NTOSKRNL_DEBUG_TIMERS
83 static int sysctl_show_timers(SYSCTL_HANDLER_ARGS);
85 SYSCTL_PROC(_debug, OID_AUTO, ntoskrnl_timers,
86 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0,
87 sysctl_show_timers, "I",
88 "Show ntoskrnl timer stats");
102 typedef struct kdpc_queue kdpc_queue;
106 struct thread *we_td;
109 typedef struct wb_ext wb_ext;
111 #define NTOSKRNL_TIMEOUTS 256
112 #ifdef NTOSKRNL_DEBUG_TIMERS
113 static uint64_t ntoskrnl_timer_fires;
114 static uint64_t ntoskrnl_timer_sets;
115 static uint64_t ntoskrnl_timer_reloads;
116 static uint64_t ntoskrnl_timer_cancels;
119 struct callout_entry {
120 struct callout ce_callout;
124 typedef struct callout_entry callout_entry;
126 static struct list_entry ntoskrnl_calllist;
127 static struct mtx ntoskrnl_calllock;
128 struct kuser_shared_data kuser_shared_data;
130 static struct list_entry ntoskrnl_intlist;
131 static kspin_lock ntoskrnl_intlock;
133 static uint8_t RtlEqualUnicodeString(unicode_string *,
134 unicode_string *, uint8_t);
135 static void RtlCopyString(ansi_string *, const ansi_string *);
136 static void RtlCopyUnicodeString(unicode_string *,
138 static irp *IoBuildSynchronousFsdRequest(uint32_t, device_object *,
139 void *, uint32_t, uint64_t *, nt_kevent *, io_status_block *);
140 static irp *IoBuildAsynchronousFsdRequest(uint32_t,
141 device_object *, void *, uint32_t, uint64_t *, io_status_block *);
142 static irp *IoBuildDeviceIoControlRequest(uint32_t,
143 device_object *, void *, uint32_t, void *, uint32_t,
144 uint8_t, nt_kevent *, io_status_block *);
145 static irp *IoAllocateIrp(uint8_t, uint8_t);
146 static void IoReuseIrp(irp *, uint32_t);
147 static void IoFreeIrp(irp *);
148 static void IoInitializeIrp(irp *, uint16_t, uint8_t);
149 static irp *IoMakeAssociatedIrp(irp *, uint8_t);
150 static uint32_t KeWaitForMultipleObjects(uint32_t,
151 nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t,
152 int64_t *, wait_block *);
153 static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t);
154 static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *);
155 static void ntoskrnl_satisfy_multiple_waits(wait_block *);
156 static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *);
157 static void ntoskrnl_insert_timer(ktimer *, int);
158 static void ntoskrnl_remove_timer(ktimer *);
159 #ifdef NTOSKRNL_DEBUG_TIMERS
160 static void ntoskrnl_show_timers(void);
162 static void ntoskrnl_timercall(void *);
163 static void ntoskrnl_dpc_thread(void *);
164 static void ntoskrnl_destroy_dpc_threads(void);
165 static void ntoskrnl_destroy_workitem_threads(void);
166 static void ntoskrnl_workitem_thread(void *);
167 static void ntoskrnl_workitem(device_object *, void *);
168 static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int);
169 static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int);
170 static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *);
171 static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t);
172 static uint16_t READ_REGISTER_USHORT(uint16_t *);
173 static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t);
174 static uint32_t READ_REGISTER_ULONG(uint32_t *);
175 static void WRITE_REGISTER_UCHAR(uint8_t *, uint8_t);
176 static uint8_t READ_REGISTER_UCHAR(uint8_t *);
177 static int64_t _allmul(int64_t, int64_t);
178 static int64_t _alldiv(int64_t, int64_t);
179 static int64_t _allrem(int64_t, int64_t);
180 static int64_t _allshr(int64_t, uint8_t);
181 static int64_t _allshl(int64_t, uint8_t);
182 static uint64_t _aullmul(uint64_t, uint64_t);
183 static uint64_t _aulldiv(uint64_t, uint64_t);
184 static uint64_t _aullrem(uint64_t, uint64_t);
185 static uint64_t _aullshr(uint64_t, uint8_t);
186 static uint64_t _aullshl(uint64_t, uint8_t);
187 static slist_entry *ntoskrnl_pushsl(slist_header *, slist_entry *);
188 static void InitializeSListHead(slist_header *);
189 static slist_entry *ntoskrnl_popsl(slist_header *);
190 static void ExFreePoolWithTag(void *, uint32_t);
191 static void ExInitializePagedLookasideList(paged_lookaside_list *,
192 lookaside_alloc_func *, lookaside_free_func *,
193 uint32_t, size_t, uint32_t, uint16_t);
194 static void ExDeletePagedLookasideList(paged_lookaside_list *);
195 static void ExInitializeNPagedLookasideList(npaged_lookaside_list *,
196 lookaside_alloc_func *, lookaside_free_func *,
197 uint32_t, size_t, uint32_t, uint16_t);
198 static void ExDeleteNPagedLookasideList(npaged_lookaside_list *);
200 *ExInterlockedPushEntrySList(slist_header *,
201 slist_entry *, kspin_lock *);
203 *ExInterlockedPopEntrySList(slist_header *, kspin_lock *);
204 static uint32_t InterlockedIncrement(volatile uint32_t *);
205 static uint32_t InterlockedDecrement(volatile uint32_t *);
206 static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t);
207 static void *MmAllocateContiguousMemory(uint32_t, uint64_t);
208 static void *MmAllocateContiguousMemorySpecifyCache(uint32_t,
209 uint64_t, uint64_t, uint64_t, enum nt_caching_type);
210 static void MmFreeContiguousMemory(void *);
211 static void MmFreeContiguousMemorySpecifyCache(void *, uint32_t,
212 enum nt_caching_type);
213 static uint32_t MmSizeOfMdl(void *, size_t);
214 static void *MmMapLockedPages(mdl *, uint8_t);
215 static void *MmMapLockedPagesSpecifyCache(mdl *,
216 uint8_t, uint32_t, void *, uint32_t, uint32_t);
217 static void MmUnmapLockedPages(void *, mdl *);
218 static device_t ntoskrnl_finddev(device_t, uint64_t, struct resource **);
219 static void RtlZeroMemory(void *, size_t);
220 static void RtlSecureZeroMemory(void *, size_t);
221 static void RtlFillMemory(void *, size_t, uint8_t);
222 static void RtlMoveMemory(void *, const void *, size_t);
223 static ndis_status RtlCharToInteger(const char *, uint32_t, uint32_t *);
224 static void RtlCopyMemory(void *, const void *, size_t);
225 static size_t RtlCompareMemory(const void *, const void *, size_t);
226 static ndis_status RtlUnicodeStringToInteger(unicode_string *,
227 uint32_t, uint32_t *);
228 static int atoi (const char *);
229 static long atol (const char *);
230 static int rand(void);
231 static void srand(unsigned int);
232 static void KeQuerySystemTime(uint64_t *);
233 static uint32_t KeTickCount(void);
234 static uint8_t IoIsWdmVersionAvailable(uint8_t, uint8_t);
235 static int32_t IoOpenDeviceRegistryKey(struct device_object *, uint32_t,
237 static void ntoskrnl_thrfunc(void *);
238 static ndis_status PsCreateSystemThread(ndis_handle *,
239 uint32_t, void *, ndis_handle, void *, void *, void *);
240 static ndis_status PsTerminateSystemThread(ndis_status);
241 static ndis_status IoGetDeviceObjectPointer(unicode_string *,
242 uint32_t, void *, device_object *);
243 static ndis_status IoGetDeviceProperty(device_object *, uint32_t,
244 uint32_t, void *, uint32_t *);
245 static void KeInitializeMutex(kmutant *, uint32_t);
246 static uint32_t KeReleaseMutex(kmutant *, uint8_t);
247 static uint32_t KeReadStateMutex(kmutant *);
248 static ndis_status ObReferenceObjectByHandle(ndis_handle,
249 uint32_t, void *, uint8_t, void **, void **);
250 static void ObfDereferenceObject(void *);
251 static uint32_t ZwClose(ndis_handle);
252 static uint32_t WmiQueryTraceInformation(uint32_t, void *, uint32_t,
254 static uint32_t WmiTraceMessage(uint64_t, uint32_t, void *, uint16_t, ...);
255 static uint32_t IoWMIRegistrationControl(device_object *, uint32_t);
256 static void *ntoskrnl_memset(void *, int, size_t);
257 static void *ntoskrnl_memmove(void *, void *, size_t);
258 static void *ntoskrnl_memchr(void *, unsigned char, size_t);
259 static char *ntoskrnl_strstr(char *, char *);
260 static char *ntoskrnl_strncat(char *, char *, size_t);
261 static int ntoskrnl_toupper(int);
262 static int ntoskrnl_tolower(int);
263 static funcptr ntoskrnl_findwrap(funcptr);
264 static uint32_t DbgPrint(char *, ...);
265 static void DbgBreakPoint(void);
266 static void KeBugCheckEx(uint32_t, u_long, u_long, u_long, u_long);
267 static int32_t KeDelayExecutionThread(uint8_t, uint8_t, int64_t *);
268 static int32_t KeSetPriorityThread(struct thread *, int32_t);
269 static void dummy(void);
271 static struct mtx ntoskrnl_dispatchlock;
272 static struct mtx ntoskrnl_interlock;
273 static kspin_lock ntoskrnl_cancellock;
274 static int ntoskrnl_kth = 0;
275 static struct nt_objref_head ntoskrnl_reflist;
276 static uma_zone_t mdl_zone;
277 static uma_zone_t iw_zone;
278 static struct kdpc_queue *kq_queues;
279 static struct kdpc_queue *wq_queues;
280 static int wq_idx = 0;
285 image_patch_table *patch;
292 mtx_init(&ntoskrnl_dispatchlock,
293 "ntoskrnl dispatch lock", MTX_NDIS_LOCK, MTX_DEF|MTX_RECURSE);
294 mtx_init(&ntoskrnl_interlock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
295 KeInitializeSpinLock(&ntoskrnl_cancellock);
296 KeInitializeSpinLock(&ntoskrnl_intlock);
297 TAILQ_INIT(&ntoskrnl_reflist);
299 InitializeListHead(&ntoskrnl_calllist);
300 InitializeListHead(&ntoskrnl_intlist);
301 mtx_init(&ntoskrnl_calllock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
303 kq_queues = ExAllocatePoolWithTag(NonPagedPool,
304 #ifdef NTOSKRNL_MULTIPLE_DPCS
305 sizeof(kdpc_queue) * mp_ncpus, 0);
307 sizeof(kdpc_queue), 0);
310 if (kq_queues == NULL)
313 wq_queues = ExAllocatePoolWithTag(NonPagedPool,
314 sizeof(kdpc_queue) * WORKITEM_THREADS, 0);
316 if (wq_queues == NULL)
319 #ifdef NTOSKRNL_MULTIPLE_DPCS
320 bzero((char *)kq_queues, sizeof(kdpc_queue) * mp_ncpus);
322 bzero((char *)kq_queues, sizeof(kdpc_queue));
324 bzero((char *)wq_queues, sizeof(kdpc_queue) * WORKITEM_THREADS);
327 * Launch the DPC threads.
330 #ifdef NTOSKRNL_MULTIPLE_DPCS
331 for (i = 0; i < mp_ncpus; i++) {
333 for (i = 0; i < 1; i++) {
337 error = kproc_create(ntoskrnl_dpc_thread, kq, &p,
338 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows DPC %d", i);
340 panic("failed to launch DPC thread");
344 * Launch the workitem threads.
347 for (i = 0; i < WORKITEM_THREADS; i++) {
349 error = kproc_create(ntoskrnl_workitem_thread, kq, &p,
350 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Workitem %d", i);
352 panic("failed to launch workitem thread");
355 patch = ntoskrnl_functbl;
356 while (patch->ipt_func != NULL) {
357 windrv_wrap((funcptr)patch->ipt_func,
358 (funcptr *)&patch->ipt_wrap,
359 patch->ipt_argcnt, patch->ipt_ftype);
363 for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) {
364 e = ExAllocatePoolWithTag(NonPagedPool,
365 sizeof(callout_entry), 0);
367 panic("failed to allocate timeouts");
368 mtx_lock_spin(&ntoskrnl_calllock);
369 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
370 mtx_unlock_spin(&ntoskrnl_calllock);
374 * MDLs are supposed to be variable size (they describe
375 * buffers containing some number of pages, but we don't
376 * know ahead of time how many pages that will be). But
377 * always allocating them off the heap is very slow. As
378 * a compromise, we create an MDL UMA zone big enough to
379 * handle any buffer requiring up to 16 pages, and we
380 * use those for any MDLs for buffers of 16 pages or less
381 * in size. For buffers larger than that (which we assume
382 * will be few and far between, we allocate the MDLs off
386 mdl_zone = uma_zcreate("Windows MDL", MDL_ZONE_SIZE,
387 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
389 iw_zone = uma_zcreate("Windows WorkItem", sizeof(io_workitem),
390 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
398 image_patch_table *patch;
402 patch = ntoskrnl_functbl;
403 while (patch->ipt_func != NULL) {
404 windrv_unwrap(patch->ipt_wrap);
408 /* Stop the workitem queues. */
409 ntoskrnl_destroy_workitem_threads();
410 /* Stop the DPC queues. */
411 ntoskrnl_destroy_dpc_threads();
413 ExFreePool(kq_queues);
414 ExFreePool(wq_queues);
416 uma_zdestroy(mdl_zone);
417 uma_zdestroy(iw_zone);
419 mtx_lock_spin(&ntoskrnl_calllock);
420 while(!IsListEmpty(&ntoskrnl_calllist)) {
421 l = RemoveHeadList(&ntoskrnl_calllist);
422 e = CONTAINING_RECORD(l, callout_entry, ce_list);
423 mtx_unlock_spin(&ntoskrnl_calllock);
425 mtx_lock_spin(&ntoskrnl_calllock);
427 mtx_unlock_spin(&ntoskrnl_calllock);
429 mtx_destroy(&ntoskrnl_dispatchlock);
430 mtx_destroy(&ntoskrnl_interlock);
431 mtx_destroy(&ntoskrnl_calllock);
437 * We need to be able to reference this externally from the wrapper;
438 * GCC only generates a local implementation of memset.
441 ntoskrnl_memset(buf, ch, size)
446 return (memset(buf, ch, size));
450 ntoskrnl_memmove(dst, src, size)
455 bcopy(src, dst, size);
460 ntoskrnl_memchr(void *buf, unsigned char ch, size_t len)
463 unsigned char *p = buf;
468 } while (--len != 0);
474 ntoskrnl_strstr(s, find)
480 if ((c = *find++) != 0) {
484 if ((sc = *s++) == 0)
487 } while (strncmp(s, find, len) != 0);
493 /* Taken from libc */
495 ntoskrnl_strncat(dst, src, n)
507 if ((*d = *s++) == 0)
531 RtlEqualUnicodeString(unicode_string *str1, unicode_string *str2,
532 uint8_t caseinsensitive)
536 if (str1->us_len != str2->us_len)
539 for (i = 0; i < str1->us_len; i++) {
540 if (caseinsensitive == TRUE) {
541 if (toupper((char)(str1->us_buf[i] & 0xFF)) !=
542 toupper((char)(str2->us_buf[i] & 0xFF)))
545 if (str1->us_buf[i] != str2->us_buf[i])
554 RtlCopyString(dst, src)
556 const ansi_string *src;
558 if (src != NULL && src->as_buf != NULL && dst->as_buf != NULL) {
559 dst->as_len = min(src->as_len, dst->as_maxlen);
560 memcpy(dst->as_buf, src->as_buf, dst->as_len);
561 if (dst->as_len < dst->as_maxlen)
562 dst->as_buf[dst->as_len] = 0;
568 RtlCopyUnicodeString(dest, src)
569 unicode_string *dest;
573 if (dest->us_maxlen >= src->us_len)
574 dest->us_len = src->us_len;
576 dest->us_len = dest->us_maxlen;
577 memcpy(dest->us_buf, src->us_buf, dest->us_len);
581 ntoskrnl_ascii_to_unicode(ascii, unicode, len)
590 for (i = 0; i < len; i++) {
591 *ustr = (uint16_t)ascii[i];
597 ntoskrnl_unicode_to_ascii(unicode, ascii, len)
606 for (i = 0; i < len / 2; i++) {
607 *astr = (uint8_t)unicode[i];
613 RtlUnicodeStringToAnsiString(ansi_string *dest, unicode_string *src, uint8_t allocate)
615 if (dest == NULL || src == NULL)
616 return (STATUS_INVALID_PARAMETER);
618 dest->as_len = src->us_len / 2;
619 if (dest->as_maxlen < dest->as_len)
620 dest->as_len = dest->as_maxlen;
622 if (allocate == TRUE) {
623 dest->as_buf = ExAllocatePoolWithTag(NonPagedPool,
624 (src->us_len / 2) + 1, 0);
625 if (dest->as_buf == NULL)
626 return (STATUS_INSUFFICIENT_RESOURCES);
627 dest->as_len = dest->as_maxlen = src->us_len / 2;
629 dest->as_len = src->us_len / 2; /* XXX */
630 if (dest->as_maxlen < dest->as_len)
631 dest->as_len = dest->as_maxlen;
634 ntoskrnl_unicode_to_ascii(src->us_buf, dest->as_buf,
637 return (STATUS_SUCCESS);
641 RtlAnsiStringToUnicodeString(unicode_string *dest, ansi_string *src,
644 if (dest == NULL || src == NULL)
645 return (STATUS_INVALID_PARAMETER);
647 if (allocate == TRUE) {
648 dest->us_buf = ExAllocatePoolWithTag(NonPagedPool,
650 if (dest->us_buf == NULL)
651 return (STATUS_INSUFFICIENT_RESOURCES);
652 dest->us_len = dest->us_maxlen = strlen(src->as_buf) * 2;
654 dest->us_len = src->as_len * 2; /* XXX */
655 if (dest->us_maxlen < dest->us_len)
656 dest->us_len = dest->us_maxlen;
659 ntoskrnl_ascii_to_unicode(src->as_buf, dest->us_buf,
662 return (STATUS_SUCCESS);
666 ExAllocatePoolWithTag(pooltype, len, tag)
673 buf = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO);
681 ExFreePoolWithTag(buf, tag)
696 IoAllocateDriverObjectExtension(drv, clid, extlen, ext)
702 custom_extension *ce;
704 ce = ExAllocatePoolWithTag(NonPagedPool, sizeof(custom_extension)
708 return (STATUS_INSUFFICIENT_RESOURCES);
711 InsertTailList((&drv->dro_driverext->dre_usrext), (&ce->ce_list));
713 *ext = (void *)(ce + 1);
715 return (STATUS_SUCCESS);
719 IoGetDriverObjectExtension(drv, clid)
724 custom_extension *ce;
727 * Sanity check. Our dummy bus drivers don't have
728 * any driver extensions.
731 if (drv->dro_driverext == NULL)
734 e = drv->dro_driverext->dre_usrext.nle_flink;
735 while (e != &drv->dro_driverext->dre_usrext) {
736 ce = (custom_extension *)e;
737 if (ce->ce_clid == clid)
738 return ((void *)(ce + 1));
747 IoCreateDevice(driver_object *drv, uint32_t devextlen, unicode_string *devname,
748 uint32_t devtype, uint32_t devchars, uint8_t exclusive,
749 device_object **newdev)
753 dev = ExAllocatePoolWithTag(NonPagedPool, sizeof(device_object), 0);
755 return (STATUS_INSUFFICIENT_RESOURCES);
757 dev->do_type = devtype;
758 dev->do_drvobj = drv;
759 dev->do_currirp = NULL;
763 dev->do_devext = ExAllocatePoolWithTag(NonPagedPool,
766 if (dev->do_devext == NULL) {
768 return (STATUS_INSUFFICIENT_RESOURCES);
771 bzero(dev->do_devext, devextlen);
773 dev->do_devext = NULL;
775 dev->do_size = sizeof(device_object) + devextlen;
777 dev->do_attacheddev = NULL;
778 dev->do_nextdev = NULL;
779 dev->do_devtype = devtype;
780 dev->do_stacksize = 1;
781 dev->do_alignreq = 1;
782 dev->do_characteristics = devchars;
783 dev->do_iotimer = NULL;
784 KeInitializeEvent(&dev->do_devlock, EVENT_TYPE_SYNC, TRUE);
787 * Vpd is used for disk/tape devices,
788 * but we don't support those. (Yet.)
792 dev->do_devobj_ext = ExAllocatePoolWithTag(NonPagedPool,
793 sizeof(devobj_extension), 0);
795 if (dev->do_devobj_ext == NULL) {
796 if (dev->do_devext != NULL)
797 ExFreePool(dev->do_devext);
799 return (STATUS_INSUFFICIENT_RESOURCES);
802 dev->do_devobj_ext->dve_type = 0;
803 dev->do_devobj_ext->dve_size = sizeof(devobj_extension);
804 dev->do_devobj_ext->dve_devobj = dev;
807 * Attach this device to the driver object's list
808 * of devices. Note: this is not the same as attaching
809 * the device to the device stack. The driver's AddDevice
810 * routine must explicitly call IoAddDeviceToDeviceStack()
814 if (drv->dro_devobj == NULL) {
815 drv->dro_devobj = dev;
816 dev->do_nextdev = NULL;
818 dev->do_nextdev = drv->dro_devobj;
819 drv->dro_devobj = dev;
824 return (STATUS_SUCCESS);
836 if (dev->do_devobj_ext != NULL)
837 ExFreePool(dev->do_devobj_ext);
839 if (dev->do_devext != NULL)
840 ExFreePool(dev->do_devext);
842 /* Unlink the device from the driver's device list. */
844 prev = dev->do_drvobj->dro_devobj;
846 dev->do_drvobj->dro_devobj = dev->do_nextdev;
848 while (prev->do_nextdev != dev)
849 prev = prev->do_nextdev;
850 prev->do_nextdev = dev->do_nextdev;
857 IoGetAttachedDevice(dev)
867 while (d->do_attacheddev != NULL)
868 d = d->do_attacheddev;
874 IoBuildSynchronousFsdRequest(func, dobj, buf, len, off, event, status)
881 io_status_block *status;
885 ip = IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status);
888 ip->irp_usrevent = event;
894 IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status)
900 io_status_block *status;
903 io_stack_location *sl;
905 ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
909 ip->irp_usriostat = status;
910 ip->irp_tail.irp_overlay.irp_thread = NULL;
912 sl = IoGetNextIrpStackLocation(ip);
913 sl->isl_major = func;
917 sl->isl_devobj = dobj;
918 sl->isl_fileobj = NULL;
919 sl->isl_completionfunc = NULL;
921 ip->irp_userbuf = buf;
923 if (dobj->do_flags & DO_BUFFERED_IO) {
924 ip->irp_assoc.irp_sysbuf =
925 ExAllocatePoolWithTag(NonPagedPool, len, 0);
926 if (ip->irp_assoc.irp_sysbuf == NULL) {
930 bcopy(buf, ip->irp_assoc.irp_sysbuf, len);
933 if (dobj->do_flags & DO_DIRECT_IO) {
934 ip->irp_mdl = IoAllocateMdl(buf, len, FALSE, FALSE, ip);
935 if (ip->irp_mdl == NULL) {
936 if (ip->irp_assoc.irp_sysbuf != NULL)
937 ExFreePool(ip->irp_assoc.irp_sysbuf);
941 ip->irp_userbuf = NULL;
942 ip->irp_assoc.irp_sysbuf = NULL;
945 if (func == IRP_MJ_READ) {
946 sl->isl_parameters.isl_read.isl_len = len;
948 sl->isl_parameters.isl_read.isl_byteoff = *off;
950 sl->isl_parameters.isl_read.isl_byteoff = 0;
953 if (func == IRP_MJ_WRITE) {
954 sl->isl_parameters.isl_write.isl_len = len;
956 sl->isl_parameters.isl_write.isl_byteoff = *off;
958 sl->isl_parameters.isl_write.isl_byteoff = 0;
965 IoBuildDeviceIoControlRequest(uint32_t iocode, device_object *dobj, void *ibuf,
966 uint32_t ilen, void *obuf, uint32_t olen, uint8_t isinternal,
967 nt_kevent *event, io_status_block *status)
970 io_stack_location *sl;
973 ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
976 ip->irp_usrevent = event;
977 ip->irp_usriostat = status;
978 ip->irp_tail.irp_overlay.irp_thread = NULL;
980 sl = IoGetNextIrpStackLocation(ip);
981 sl->isl_major = isinternal == TRUE ?
982 IRP_MJ_INTERNAL_DEVICE_CONTROL : IRP_MJ_DEVICE_CONTROL;
986 sl->isl_devobj = dobj;
987 sl->isl_fileobj = NULL;
988 sl->isl_completionfunc = NULL;
989 sl->isl_parameters.isl_ioctl.isl_iocode = iocode;
990 sl->isl_parameters.isl_ioctl.isl_ibuflen = ilen;
991 sl->isl_parameters.isl_ioctl.isl_obuflen = olen;
993 switch(IO_METHOD(iocode)) {
994 case METHOD_BUFFERED:
1000 ip->irp_assoc.irp_sysbuf =
1001 ExAllocatePoolWithTag(NonPagedPool, buflen, 0);
1002 if (ip->irp_assoc.irp_sysbuf == NULL) {
1007 if (ilen && ibuf != NULL) {
1008 bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1009 bzero((char *)ip->irp_assoc.irp_sysbuf + ilen,
1012 bzero(ip->irp_assoc.irp_sysbuf, ilen);
1013 ip->irp_userbuf = obuf;
1015 case METHOD_IN_DIRECT:
1016 case METHOD_OUT_DIRECT:
1017 if (ilen && ibuf != NULL) {
1018 ip->irp_assoc.irp_sysbuf =
1019 ExAllocatePoolWithTag(NonPagedPool, ilen, 0);
1020 if (ip->irp_assoc.irp_sysbuf == NULL) {
1024 bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
1026 if (olen && obuf != NULL) {
1027 ip->irp_mdl = IoAllocateMdl(obuf, olen,
1030 * Normally we would MmProbeAndLockPages()
1031 * here, but we don't have to in our
1036 case METHOD_NEITHER:
1037 ip->irp_userbuf = obuf;
1038 sl->isl_parameters.isl_ioctl.isl_type3ibuf = ibuf;
1045 * Ideally, we should associate this IRP with the calling
1053 IoAllocateIrp(uint8_t stsize, uint8_t chargequota)
1057 i = ExAllocatePoolWithTag(NonPagedPool, IoSizeOfIrp(stsize), 0);
1061 IoInitializeIrp(i, IoSizeOfIrp(stsize), stsize);
1067 IoMakeAssociatedIrp(irp *ip, uint8_t stsize)
1071 associrp = IoAllocateIrp(stsize, FALSE);
1072 if (associrp == NULL)
1075 mtx_lock(&ntoskrnl_dispatchlock);
1076 associrp->irp_flags |= IRP_ASSOCIATED_IRP;
1077 associrp->irp_tail.irp_overlay.irp_thread =
1078 ip->irp_tail.irp_overlay.irp_thread;
1079 associrp->irp_assoc.irp_master = ip;
1080 mtx_unlock(&ntoskrnl_dispatchlock);
1093 IoInitializeIrp(irp *io, uint16_t psize, uint8_t ssize)
1095 bzero((char *)io, IoSizeOfIrp(ssize));
1096 io->irp_size = psize;
1097 io->irp_stackcnt = ssize;
1098 io->irp_currentstackloc = ssize;
1099 InitializeListHead(&io->irp_thlist);
1100 io->irp_tail.irp_overlay.irp_csl =
1101 (io_stack_location *)(io + 1) + ssize;
1105 IoReuseIrp(ip, status)
1111 allocflags = ip->irp_allocflags;
1112 IoInitializeIrp(ip, ip->irp_size, ip->irp_stackcnt);
1113 ip->irp_iostat.isb_status = status;
1114 ip->irp_allocflags = allocflags;
1118 IoAcquireCancelSpinLock(uint8_t *irql)
1120 KeAcquireSpinLock(&ntoskrnl_cancellock, irql);
1124 IoReleaseCancelSpinLock(uint8_t irql)
1126 KeReleaseSpinLock(&ntoskrnl_cancellock, irql);
1130 IoCancelIrp(irp *ip)
1135 IoAcquireCancelSpinLock(&cancelirql);
1136 cfunc = IoSetCancelRoutine(ip, NULL);
1137 ip->irp_cancel = TRUE;
1138 if (cfunc == NULL) {
1139 IoReleaseCancelSpinLock(cancelirql);
1142 ip->irp_cancelirql = cancelirql;
1143 MSCALL2(cfunc, IoGetCurrentIrpStackLocation(ip)->isl_devobj, ip);
1144 return (uint8_t)IoSetCancelValue(ip, TRUE);
1148 IofCallDriver(dobj, ip)
1149 device_object *dobj;
1152 driver_object *drvobj;
1153 io_stack_location *sl;
1155 driver_dispatch disp;
1157 drvobj = dobj->do_drvobj;
1159 if (ip->irp_currentstackloc <= 0)
1160 panic("IoCallDriver(): out of stack locations");
1162 IoSetNextIrpStackLocation(ip);
1163 sl = IoGetCurrentIrpStackLocation(ip);
1165 sl->isl_devobj = dobj;
1167 disp = drvobj->dro_dispatch[sl->isl_major];
1168 status = MSCALL2(disp, dobj, ip);
1174 IofCompleteRequest(irp *ip, uint8_t prioboost)
1177 device_object *dobj;
1178 io_stack_location *sl;
1181 KASSERT(ip->irp_iostat.isb_status != STATUS_PENDING,
1182 ("incorrect IRP(%p) status (STATUS_PENDING)", ip));
1184 sl = IoGetCurrentIrpStackLocation(ip);
1185 IoSkipCurrentIrpStackLocation(ip);
1188 if (sl->isl_ctl & SL_PENDING_RETURNED)
1189 ip->irp_pendingreturned = TRUE;
1191 if (ip->irp_currentstackloc != (ip->irp_stackcnt + 1))
1192 dobj = IoGetCurrentIrpStackLocation(ip)->isl_devobj;
1196 if (sl->isl_completionfunc != NULL &&
1197 ((ip->irp_iostat.isb_status == STATUS_SUCCESS &&
1198 sl->isl_ctl & SL_INVOKE_ON_SUCCESS) ||
1199 (ip->irp_iostat.isb_status != STATUS_SUCCESS &&
1200 sl->isl_ctl & SL_INVOKE_ON_ERROR) ||
1201 (ip->irp_cancel == TRUE &&
1202 sl->isl_ctl & SL_INVOKE_ON_CANCEL))) {
1203 cf = sl->isl_completionfunc;
1204 status = MSCALL3(cf, dobj, ip, sl->isl_completionctx);
1205 if (status == STATUS_MORE_PROCESSING_REQUIRED)
1208 if ((ip->irp_currentstackloc <= ip->irp_stackcnt) &&
1209 (ip->irp_pendingreturned == TRUE))
1210 IoMarkIrpPending(ip);
1213 /* move to the next. */
1214 IoSkipCurrentIrpStackLocation(ip);
1216 } while (ip->irp_currentstackloc <= (ip->irp_stackcnt + 1));
1218 if (ip->irp_usriostat != NULL)
1219 *ip->irp_usriostat = ip->irp_iostat;
1220 if (ip->irp_usrevent != NULL)
1221 KeSetEvent(ip->irp_usrevent, prioboost, FALSE);
1223 /* Handle any associated IRPs. */
1225 if (ip->irp_flags & IRP_ASSOCIATED_IRP) {
1226 uint32_t masterirpcnt;
1230 masterirp = ip->irp_assoc.irp_master;
1232 InterlockedDecrement(&masterirp->irp_assoc.irp_irpcnt);
1234 while ((m = ip->irp_mdl) != NULL) {
1235 ip->irp_mdl = m->mdl_next;
1239 if (masterirpcnt == 0)
1240 IoCompleteRequest(masterirp, IO_NO_INCREMENT);
1244 /* With any luck, these conditions will never arise. */
1246 if (ip->irp_flags & IRP_PAGING_IO) {
1247 if (ip->irp_mdl != NULL)
1248 IoFreeMdl(ip->irp_mdl);
1262 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1263 l = ntoskrnl_intlist.nle_flink;
1264 while (l != &ntoskrnl_intlist) {
1265 iobj = CONTAINING_RECORD(l, kinterrupt, ki_list);
1266 claimed = MSCALL2(iobj->ki_svcfunc, iobj, iobj->ki_svcctx);
1267 if (claimed == TRUE)
1271 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1275 KeAcquireInterruptSpinLock(iobj)
1279 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1284 KeReleaseInterruptSpinLock(kinterrupt *iobj, uint8_t irql)
1286 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1290 KeSynchronizeExecution(iobj, syncfunc, syncctx)
1297 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1298 MSCALL1(syncfunc, syncctx);
1299 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1305 * IoConnectInterrupt() is passed only the interrupt vector and
1306 * irql that a device wants to use, but no device-specific tag
1307 * of any kind. This conflicts rather badly with FreeBSD's
1308 * bus_setup_intr(), which needs the device_t for the device
1309 * requesting interrupt delivery. In order to bypass this
1310 * inconsistency, we implement a second level of interrupt
1311 * dispatching on top of bus_setup_intr(). All devices use
1312 * ntoskrnl_intr() as their ISR, and any device requesting
1313 * interrupts will be registered with ntoskrnl_intr()'s interrupt
1314 * dispatch list. When an interrupt arrives, we walk the list
1315 * and invoke all the registered ISRs. This effectively makes all
1316 * interrupts shared, but it's the only way to duplicate the
1317 * semantics of IoConnectInterrupt() and IoDisconnectInterrupt() properly.
1321 IoConnectInterrupt(kinterrupt **iobj, void *svcfunc, void *svcctx,
1322 kspin_lock *lock, uint32_t vector, uint8_t irql, uint8_t syncirql,
1323 uint8_t imode, uint8_t shared, uint32_t affinity, uint8_t savefloat)
1327 *iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0);
1329 return (STATUS_INSUFFICIENT_RESOURCES);
1331 (*iobj)->ki_svcfunc = svcfunc;
1332 (*iobj)->ki_svcctx = svcctx;
1335 KeInitializeSpinLock(&(*iobj)->ki_lock_priv);
1336 (*iobj)->ki_lock = &(*iobj)->ki_lock_priv;
1338 (*iobj)->ki_lock = lock;
1340 KeAcquireSpinLock(&ntoskrnl_intlock, &curirql);
1341 InsertHeadList((&ntoskrnl_intlist), (&(*iobj)->ki_list));
1342 KeReleaseSpinLock(&ntoskrnl_intlock, curirql);
1344 return (STATUS_SUCCESS);
1348 IoDisconnectInterrupt(iobj)
1356 KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
1357 RemoveEntryList((&iobj->ki_list));
1358 KeReleaseSpinLock(&ntoskrnl_intlock, irql);
1364 IoAttachDeviceToDeviceStack(src, dst)
1368 device_object *attached;
1370 mtx_lock(&ntoskrnl_dispatchlock);
1371 attached = IoGetAttachedDevice(dst);
1372 attached->do_attacheddev = src;
1373 src->do_attacheddev = NULL;
1374 src->do_stacksize = attached->do_stacksize + 1;
1375 mtx_unlock(&ntoskrnl_dispatchlock);
1381 IoDetachDevice(topdev)
1382 device_object *topdev;
1384 device_object *tail;
1386 mtx_lock(&ntoskrnl_dispatchlock);
1388 /* First, break the chain. */
1389 tail = topdev->do_attacheddev;
1391 mtx_unlock(&ntoskrnl_dispatchlock);
1394 topdev->do_attacheddev = tail->do_attacheddev;
1395 topdev->do_refcnt--;
1397 /* Now reduce the stacksize count for the takm_il objects. */
1399 tail = topdev->do_attacheddev;
1400 while (tail != NULL) {
1401 tail->do_stacksize--;
1402 tail = tail->do_attacheddev;
1405 mtx_unlock(&ntoskrnl_dispatchlock);
1409 * For the most part, an object is considered signalled if
1410 * dh_sigstate == TRUE. The exception is for mutant objects
1411 * (mutexes), where the logic works like this:
1413 * - If the thread already owns the object and sigstate is
1414 * less than or equal to 0, then the object is considered
1415 * signalled (recursive acquisition).
1416 * - If dh_sigstate == 1, the object is also considered
1421 ntoskrnl_is_signalled(obj, td)
1422 nt_dispatch_header *obj;
1427 if (obj->dh_type == DISP_TYPE_MUTANT) {
1428 km = (kmutant *)obj;
1429 if ((obj->dh_sigstate <= 0 && km->km_ownerthread == td) ||
1430 obj->dh_sigstate == 1)
1435 if (obj->dh_sigstate > 0)
1441 ntoskrnl_satisfy_wait(obj, td)
1442 nt_dispatch_header *obj;
1447 switch (obj->dh_type) {
1448 case DISP_TYPE_MUTANT:
1449 km = (struct kmutant *)obj;
1452 * If sigstate reaches 0, the mutex is now
1453 * non-signalled (the new thread owns it).
1455 if (obj->dh_sigstate == 0) {
1456 km->km_ownerthread = td;
1457 if (km->km_abandoned == TRUE)
1458 km->km_abandoned = FALSE;
1461 /* Synchronization objects get reset to unsignalled. */
1462 case DISP_TYPE_SYNCHRONIZATION_EVENT:
1463 case DISP_TYPE_SYNCHRONIZATION_TIMER:
1464 obj->dh_sigstate = 0;
1466 case DISP_TYPE_SEMAPHORE:
1475 ntoskrnl_satisfy_multiple_waits(wb)
1482 td = wb->wb_kthread;
1485 ntoskrnl_satisfy_wait(wb->wb_object, td);
1486 cur->wb_awakened = TRUE;
1488 } while (cur != wb);
1491 /* Always called with dispatcher lock held. */
1493 ntoskrnl_waittest(obj, increment)
1494 nt_dispatch_header *obj;
1497 wait_block *w, *next;
1504 * Once an object has been signalled, we walk its list of
1505 * wait blocks. If a wait block can be awakened, then satisfy
1506 * waits as necessary and wake the thread.
1508 * The rules work like this:
1510 * If a wait block is marked as WAITTYPE_ANY, then
1511 * we can satisfy the wait conditions on the current
1512 * object and wake the thread right away. Satisfying
1513 * the wait also has the effect of breaking us out
1514 * of the search loop.
1516 * If the object is marked as WAITTYLE_ALL, then the
1517 * wait block will be part of a circularly linked
1518 * list of wait blocks belonging to a waiting thread
1519 * that's sleeping in KeWaitForMultipleObjects(). In
1520 * order to wake the thread, all the objects in the
1521 * wait list must be in the signalled state. If they
1522 * are, we then satisfy all of them and wake the
1527 e = obj->dh_waitlisthead.nle_flink;
1529 while (e != &obj->dh_waitlisthead && obj->dh_sigstate > 0) {
1530 w = CONTAINING_RECORD(e, wait_block, wb_waitlist);
1534 if (w->wb_waittype == WAITTYPE_ANY) {
1536 * Thread can be awakened if
1537 * any wait is satisfied.
1539 ntoskrnl_satisfy_wait(obj, td);
1541 w->wb_awakened = TRUE;
1544 * Thread can only be woken up
1545 * if all waits are satisfied.
1546 * If the thread is waiting on multiple
1547 * objects, they should all be linked
1548 * through the wb_next pointers in the
1554 if (ntoskrnl_is_signalled(obj, td) == FALSE) {
1558 next = next->wb_next;
1560 ntoskrnl_satisfy_multiple_waits(w);
1563 if (satisfied == TRUE)
1564 cv_broadcastpri(&we->we_cv,
1565 (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
1566 w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
1573 * Return the number of 100 nanosecond intervals since
1574 * January 1, 1601. (?!?!)
1583 *tval = (uint64_t)ts.tv_nsec / 100 + (uint64_t)ts.tv_sec * 10000000 +
1584 11644473600 * 10000000; /* 100ns ticks from 1601 to 1970 */
1588 KeQuerySystemTime(current_time)
1589 uint64_t *current_time;
1591 ntoskrnl_time(current_time);
1598 getmicrouptime(&tv);
1604 * KeWaitForSingleObject() is a tricky beast, because it can be used
1605 * with several different object types: semaphores, timers, events,
1606 * mutexes and threads. Semaphores don't appear very often, but the
1607 * other object types are quite common. KeWaitForSingleObject() is
1608 * what's normally used to acquire a mutex, and it can be used to
1609 * wait for a thread termination.
1611 * The Windows NDIS API is implemented in terms of Windows kernel
1612 * primitives, and some of the object manipulation is duplicated in
1613 * NDIS. For example, NDIS has timers and events, which are actually
1614 * Windows kevents and ktimers. Now, you're supposed to only use the
1615 * NDIS variants of these objects within the confines of the NDIS API,
1616 * but there are some naughty developers out there who will use
1617 * KeWaitForSingleObject() on NDIS timer and event objects, so we
1618 * have to support that as well. Conseqently, our NDIS timer and event
1619 * code has to be closely tied into our ntoskrnl timer and event code,
1620 * just as it is in Windows.
1622 * KeWaitForSingleObject() may do different things for different kinds
1625 * - For events, we check if the event has been signalled. If the
1626 * event is already in the signalled state, we just return immediately,
1627 * otherwise we wait for it to be set to the signalled state by someone
1628 * else calling KeSetEvent(). Events can be either synchronization or
1629 * notification events.
1631 * - For timers, if the timer has already fired and the timer is in
1632 * the signalled state, we just return, otherwise we wait on the
1633 * timer. Unlike an event, timers get signalled automatically when
1634 * they expire rather than someone having to trip them manually.
1635 * Timers initialized with KeInitializeTimer() are always notification
1636 * events: KeInitializeTimerEx() lets you initialize a timer as
1637 * either a notification or synchronization event.
1639 * - For mutexes, we try to acquire the mutex and if we can't, we wait
1640 * on the mutex until it's available and then grab it. When a mutex is
1641 * released, it enters the signalled state, which wakes up one of the
1642 * threads waiting to acquire it. Mutexes are always synchronization
1645 * - For threads, the only thing we do is wait until the thread object
1646 * enters a signalled state, which occurs when the thread terminates.
1647 * Threads are always notification events.
1649 * A notification event wakes up all threads waiting on an object. A
1650 * synchronization event wakes up just one. Also, a synchronization event
1651 * is auto-clearing, which means we automatically set the event back to
1652 * the non-signalled state once the wakeup is done.
1656 KeWaitForSingleObject(void *arg, uint32_t reason, uint32_t mode,
1657 uint8_t alertable, int64_t *duetime)
1660 struct thread *td = curthread;
1665 nt_dispatch_header *obj;
1670 return (STATUS_INVALID_PARAMETER);
1672 mtx_lock(&ntoskrnl_dispatchlock);
1674 cv_init(&we.we_cv, "KeWFS");
1678 * Check to see if this object is already signalled,
1679 * and just return without waiting if it is.
1681 if (ntoskrnl_is_signalled(obj, td) == TRUE) {
1682 /* Sanity check the signal state value. */
1683 if (obj->dh_sigstate != INT32_MIN) {
1684 ntoskrnl_satisfy_wait(obj, curthread);
1685 mtx_unlock(&ntoskrnl_dispatchlock);
1686 return (STATUS_SUCCESS);
1689 * There's a limit to how many times we can
1690 * recursively acquire a mutant. If we hit
1691 * the limit, something is very wrong.
1693 if (obj->dh_type == DISP_TYPE_MUTANT) {
1694 mtx_unlock(&ntoskrnl_dispatchlock);
1695 panic("mutant limit exceeded");
1700 bzero((char *)&w, sizeof(wait_block));
1703 w.wb_waittype = WAITTYPE_ANY;
1706 w.wb_awakened = FALSE;
1707 w.wb_oldpri = td->td_priority;
1709 InsertTailList((&obj->dh_waitlisthead), (&w.wb_waitlist));
1712 * The timeout value is specified in 100 nanosecond units
1713 * and can be a positive or negative number. If it's positive,
1714 * then the duetime is absolute, and we need to convert it
1715 * to an absolute offset relative to now in order to use it.
1716 * If it's negative, then the duetime is relative and we
1717 * just have to convert the units.
1720 if (duetime != NULL) {
1722 tv.tv_sec = - (*duetime) / 10000000;
1723 tv.tv_usec = (- (*duetime) / 10) -
1724 (tv.tv_sec * 1000000);
1726 ntoskrnl_time(&curtime);
1727 if (*duetime < curtime)
1728 tv.tv_sec = tv.tv_usec = 0;
1730 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1731 tv.tv_usec = ((*duetime) - curtime) / 10 -
1732 (tv.tv_sec * 1000000);
1737 if (duetime == NULL)
1738 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1740 error = cv_timedwait(&we.we_cv,
1741 &ntoskrnl_dispatchlock, tvtohz(&tv));
1743 RemoveEntryList(&w.wb_waitlist);
1745 cv_destroy(&we.we_cv);
1747 /* We timed out. Leave the object alone and return status. */
1749 if (error == EWOULDBLOCK) {
1750 mtx_unlock(&ntoskrnl_dispatchlock);
1751 return (STATUS_TIMEOUT);
1754 mtx_unlock(&ntoskrnl_dispatchlock);
1756 return (STATUS_SUCCESS);
1758 return (KeWaitForMultipleObjects(1, &obj, WAITTYPE_ALL, reason,
1759 mode, alertable, duetime, &w));
1764 KeWaitForMultipleObjects(uint32_t cnt, nt_dispatch_header *obj[], uint32_t wtype,
1765 uint32_t reason, uint32_t mode, uint8_t alertable, int64_t *duetime,
1766 wait_block *wb_array)
1768 struct thread *td = curthread;
1769 wait_block *whead, *w;
1770 wait_block _wb_array[MAX_WAIT_OBJECTS];
1771 nt_dispatch_header *cur;
1773 int i, wcnt = 0, error = 0;
1775 struct timespec t1, t2;
1776 uint32_t status = STATUS_SUCCESS;
1779 if (cnt > MAX_WAIT_OBJECTS)
1780 return (STATUS_INVALID_PARAMETER);
1781 if (cnt > THREAD_WAIT_OBJECTS && wb_array == NULL)
1782 return (STATUS_INVALID_PARAMETER);
1784 mtx_lock(&ntoskrnl_dispatchlock);
1786 cv_init(&we.we_cv, "KeWFM");
1789 if (wb_array == NULL)
1794 bzero((char *)whead, sizeof(wait_block) * cnt);
1796 /* First pass: see if we can satisfy any waits immediately. */
1801 for (i = 0; i < cnt; i++) {
1802 InsertTailList((&obj[i]->dh_waitlisthead),
1805 w->wb_object = obj[i];
1806 w->wb_waittype = wtype;
1808 w->wb_awakened = FALSE;
1809 w->wb_oldpri = td->td_priority;
1813 if (ntoskrnl_is_signalled(obj[i], td)) {
1815 * There's a limit to how many times
1816 * we can recursively acquire a mutant.
1817 * If we hit the limit, something
1820 if (obj[i]->dh_sigstate == INT32_MIN &&
1821 obj[i]->dh_type == DISP_TYPE_MUTANT) {
1822 mtx_unlock(&ntoskrnl_dispatchlock);
1823 panic("mutant limit exceeded");
1827 * If this is a WAITTYPE_ANY wait, then
1828 * satisfy the waited object and exit
1832 if (wtype == WAITTYPE_ANY) {
1833 ntoskrnl_satisfy_wait(obj[i], td);
1834 status = STATUS_WAIT_0 + i;
1839 w->wb_object = NULL;
1840 RemoveEntryList(&w->wb_waitlist);
1846 * If this is a WAITTYPE_ALL wait and all objects are
1847 * already signalled, satisfy the waits and exit now.
1850 if (wtype == WAITTYPE_ALL && wcnt == 0) {
1851 for (i = 0; i < cnt; i++)
1852 ntoskrnl_satisfy_wait(obj[i], td);
1853 status = STATUS_SUCCESS;
1858 * Create a circular waitblock list. The waitcount
1859 * must always be non-zero when we get here.
1862 (w - 1)->wb_next = whead;
1864 /* Wait on any objects that aren't yet signalled. */
1866 /* Calculate timeout, if any. */
1868 if (duetime != NULL) {
1870 tv.tv_sec = - (*duetime) / 10000000;
1871 tv.tv_usec = (- (*duetime) / 10) -
1872 (tv.tv_sec * 1000000);
1874 ntoskrnl_time(&curtime);
1875 if (*duetime < curtime)
1876 tv.tv_sec = tv.tv_usec = 0;
1878 tv.tv_sec = ((*duetime) - curtime) / 10000000;
1879 tv.tv_usec = ((*duetime) - curtime) / 10 -
1880 (tv.tv_sec * 1000000);
1888 if (duetime == NULL)
1889 cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
1891 error = cv_timedwait(&we.we_cv,
1892 &ntoskrnl_dispatchlock, tvtohz(&tv));
1894 /* Wait with timeout expired. */
1897 status = STATUS_TIMEOUT;
1903 /* See what's been signalled. */
1908 if (ntoskrnl_is_signalled(cur, td) == TRUE ||
1909 w->wb_awakened == TRUE) {
1910 /* Sanity check the signal state value. */
1911 if (cur->dh_sigstate == INT32_MIN &&
1912 cur->dh_type == DISP_TYPE_MUTANT) {
1913 mtx_unlock(&ntoskrnl_dispatchlock);
1914 panic("mutant limit exceeded");
1917 if (wtype == WAITTYPE_ANY) {
1918 status = w->wb_waitkey &
1924 } while (w != whead);
1927 * If all objects have been signalled, or if this
1928 * is a WAITTYPE_ANY wait and we were woke up by
1929 * someone, we can bail.
1933 status = STATUS_SUCCESS;
1938 * If this is WAITTYPE_ALL wait, and there's still
1939 * objects that haven't been signalled, deduct the
1940 * time that's elapsed so far from the timeout and
1941 * wait again (or continue waiting indefinitely if
1942 * there's no timeout).
1945 if (duetime != NULL) {
1946 tv.tv_sec -= (t2.tv_sec - t1.tv_sec);
1947 tv.tv_usec -= (t2.tv_nsec - t1.tv_nsec) / 1000;
1954 cv_destroy(&we.we_cv);
1956 for (i = 0; i < cnt; i++) {
1957 if (whead[i].wb_object != NULL)
1958 RemoveEntryList(&whead[i].wb_waitlist);
1961 mtx_unlock(&ntoskrnl_dispatchlock);
1967 WRITE_REGISTER_USHORT(uint16_t *reg, uint16_t val)
1969 bus_space_write_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1973 READ_REGISTER_USHORT(reg)
1976 return (bus_space_read_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1980 WRITE_REGISTER_ULONG(reg, val)
1984 bus_space_write_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
1988 READ_REGISTER_ULONG(reg)
1991 return (bus_space_read_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
1995 READ_REGISTER_UCHAR(uint8_t *reg)
1997 return (bus_space_read_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
2001 WRITE_REGISTER_UCHAR(uint8_t *reg, uint8_t val)
2003 bus_space_write_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
2055 _allshl(int64_t a, uint8_t b)
2061 _aullshl(uint64_t a, uint8_t b)
2067 _allshr(int64_t a, uint8_t b)
2073 _aullshr(uint64_t a, uint8_t b)
2078 static slist_entry *
2079 ntoskrnl_pushsl(head, entry)
2083 slist_entry *oldhead;
2085 oldhead = head->slh_list.slh_next;
2086 entry->sl_next = head->slh_list.slh_next;
2087 head->slh_list.slh_next = entry;
2088 head->slh_list.slh_depth++;
2089 head->slh_list.slh_seq++;
2095 InitializeSListHead(head)
2098 memset(head, 0, sizeof(*head));
2101 static slist_entry *
2102 ntoskrnl_popsl(head)
2107 first = head->slh_list.slh_next;
2108 if (first != NULL) {
2109 head->slh_list.slh_next = first->sl_next;
2110 head->slh_list.slh_depth--;
2111 head->slh_list.slh_seq++;
2118 * We need this to make lookaside lists work for amd64.
2119 * We pass a pointer to ExAllocatePoolWithTag() the lookaside
2120 * list structure. For amd64 to work right, this has to be a
2121 * pointer to the wrapped version of the routine, not the
2122 * original. Letting the Windows driver invoke the original
2123 * function directly will result in a convention calling
2124 * mismatch and a pretty crash. On x86, this effectively
2125 * becomes a no-op since ipt_func and ipt_wrap are the same.
2129 ntoskrnl_findwrap(func)
2132 image_patch_table *patch;
2134 patch = ntoskrnl_functbl;
2135 while (patch->ipt_func != NULL) {
2136 if ((funcptr)patch->ipt_func == func)
2137 return ((funcptr)patch->ipt_wrap);
2145 ExInitializePagedLookasideList(paged_lookaside_list *lookaside,
2146 lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2147 uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2149 bzero((char *)lookaside, sizeof(paged_lookaside_list));
2151 if (size < sizeof(slist_entry))
2152 lookaside->nll_l.gl_size = sizeof(slist_entry);
2154 lookaside->nll_l.gl_size = size;
2155 lookaside->nll_l.gl_tag = tag;
2156 if (allocfunc == NULL)
2157 lookaside->nll_l.gl_allocfunc =
2158 ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2160 lookaside->nll_l.gl_allocfunc = allocfunc;
2162 if (freefunc == NULL)
2163 lookaside->nll_l.gl_freefunc =
2164 ntoskrnl_findwrap((funcptr)ExFreePool);
2166 lookaside->nll_l.gl_freefunc = freefunc;
2169 KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2172 lookaside->nll_l.gl_type = NonPagedPool;
2173 lookaside->nll_l.gl_depth = depth;
2174 lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2178 ExDeletePagedLookasideList(lookaside)
2179 paged_lookaside_list *lookaside;
2182 void (*freefunc)(void *);
2184 freefunc = lookaside->nll_l.gl_freefunc;
2185 while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2186 MSCALL1(freefunc, buf);
2190 ExInitializeNPagedLookasideList(npaged_lookaside_list *lookaside,
2191 lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
2192 uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
2194 bzero((char *)lookaside, sizeof(npaged_lookaside_list));
2196 if (size < sizeof(slist_entry))
2197 lookaside->nll_l.gl_size = sizeof(slist_entry);
2199 lookaside->nll_l.gl_size = size;
2200 lookaside->nll_l.gl_tag = tag;
2201 if (allocfunc == NULL)
2202 lookaside->nll_l.gl_allocfunc =
2203 ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
2205 lookaside->nll_l.gl_allocfunc = allocfunc;
2207 if (freefunc == NULL)
2208 lookaside->nll_l.gl_freefunc =
2209 ntoskrnl_findwrap((funcptr)ExFreePool);
2211 lookaside->nll_l.gl_freefunc = freefunc;
2214 KeInitializeSpinLock(&lookaside->nll_obsoletelock);
2217 lookaside->nll_l.gl_type = NonPagedPool;
2218 lookaside->nll_l.gl_depth = depth;
2219 lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
2223 ExDeleteNPagedLookasideList(lookaside)
2224 npaged_lookaside_list *lookaside;
2227 void (*freefunc)(void *);
2229 freefunc = lookaside->nll_l.gl_freefunc;
2230 while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
2231 MSCALL1(freefunc, buf);
2235 InterlockedPushEntrySList(head, entry)
2239 slist_entry *oldhead;
2241 mtx_lock_spin(&ntoskrnl_interlock);
2242 oldhead = ntoskrnl_pushsl(head, entry);
2243 mtx_unlock_spin(&ntoskrnl_interlock);
2249 InterlockedPopEntrySList(head)
2254 mtx_lock_spin(&ntoskrnl_interlock);
2255 first = ntoskrnl_popsl(head);
2256 mtx_unlock_spin(&ntoskrnl_interlock);
2261 static slist_entry *
2262 ExInterlockedPushEntrySList(head, entry, lock)
2267 return (InterlockedPushEntrySList(head, entry));
2270 static slist_entry *
2271 ExInterlockedPopEntrySList(head, lock)
2275 return (InterlockedPopEntrySList(head));
2279 ExQueryDepthSList(head)
2284 mtx_lock_spin(&ntoskrnl_interlock);
2285 depth = head->slh_list.slh_depth;
2286 mtx_unlock_spin(&ntoskrnl_interlock);
2292 KeInitializeSpinLock(lock)
2300 KefAcquireSpinLockAtDpcLevel(lock)
2303 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2307 while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) {
2309 #ifdef NTOSKRNL_DEBUG_SPINLOCKS
2318 KefReleaseSpinLockFromDpcLevel(lock)
2321 atomic_store_rel_int((volatile u_int *)lock, 0);
2325 KeAcquireSpinLockRaiseToDpc(kspin_lock *lock)
2329 if (KeGetCurrentIrql() > DISPATCH_LEVEL)
2330 panic("IRQL_NOT_LESS_THAN_OR_EQUAL");
2332 KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
2333 KeAcquireSpinLockAtDpcLevel(lock);
2339 KeAcquireSpinLockAtDpcLevel(kspin_lock *lock)
2341 while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0)
2346 KeReleaseSpinLockFromDpcLevel(kspin_lock *lock)
2348 atomic_store_rel_int((volatile u_int *)lock, 0);
2350 #endif /* __i386__ */
2353 InterlockedExchange(dst, val)
2354 volatile uint32_t *dst;
2359 mtx_lock_spin(&ntoskrnl_interlock);
2362 mtx_unlock_spin(&ntoskrnl_interlock);
2368 InterlockedIncrement(addend)
2369 volatile uint32_t *addend;
2371 atomic_add_long((volatile u_long *)addend, 1);
2376 InterlockedDecrement(addend)
2377 volatile uint32_t *addend;
2379 atomic_subtract_long((volatile u_long *)addend, 1);
2384 ExInterlockedAddLargeStatistic(addend, inc)
2388 mtx_lock_spin(&ntoskrnl_interlock);
2390 mtx_unlock_spin(&ntoskrnl_interlock);
2394 IoAllocateMdl(void *vaddr, uint32_t len, uint8_t secondarybuf,
2395 uint8_t chargequota, irp *iopkt)
2400 if (MmSizeOfMdl(vaddr, len) > MDL_ZONE_SIZE)
2401 m = ExAllocatePoolWithTag(NonPagedPool,
2402 MmSizeOfMdl(vaddr, len), 0);
2404 m = uma_zalloc(mdl_zone, M_NOWAIT | M_ZERO);
2411 MmInitializeMdl(m, vaddr, len);
2414 * MmInitializMdl() clears the flags field, so we
2415 * have to set this here. If the MDL came from the
2416 * MDL UMA zone, tag it so we can release it to
2417 * the right place later.
2420 m->mdl_flags = MDL_ZONE_ALLOCED;
2422 if (iopkt != NULL) {
2423 if (secondarybuf == TRUE) {
2425 last = iopkt->irp_mdl;
2426 while (last->mdl_next != NULL)
2427 last = last->mdl_next;
2430 if (iopkt->irp_mdl != NULL)
2431 panic("leaking an MDL in IoAllocateMdl()");
2446 if (m->mdl_flags & MDL_ZONE_ALLOCED)
2447 uma_zfree(mdl_zone, m);
2453 MmAllocateContiguousMemory(size, highest)
2458 size_t pagelength = roundup(size, PAGE_SIZE);
2460 addr = ExAllocatePoolWithTag(NonPagedPool, pagelength, 0);
2466 MmAllocateContiguousMemorySpecifyCache(size, lowest, highest,
2467 boundary, cachetype)
2472 enum nt_caching_type cachetype;
2474 vm_memattr_t memattr;
2477 switch (cachetype) {
2479 memattr = VM_MEMATTR_UNCACHEABLE;
2481 case MmWriteCombined:
2482 memattr = VM_MEMATTR_WRITE_COMBINING;
2484 case MmNonCachedUnordered:
2485 memattr = VM_MEMATTR_UNCACHEABLE;
2488 case MmHardwareCoherentCached:
2491 memattr = VM_MEMATTR_DEFAULT;
2495 ret = (void *)kmem_alloc_contig(size, M_ZERO | M_NOWAIT, lowest,
2496 highest, PAGE_SIZE, boundary, memattr);
2498 malloc_type_allocated(M_DEVBUF, round_page(size));
2503 MmFreeContiguousMemory(base)
2510 MmFreeContiguousMemorySpecifyCache(base, size, cachetype)
2513 enum nt_caching_type cachetype;
2515 contigfree(base, size, M_DEVBUF);
2519 MmSizeOfMdl(vaddr, len)
2525 l = sizeof(struct mdl) +
2526 (sizeof(vm_offset_t *) * SPAN_PAGES(vaddr, len));
2532 * The Microsoft documentation says this routine fills in the
2533 * page array of an MDL with the _physical_ page addresses that
2534 * comprise the buffer, but we don't really want to do that here.
2535 * Instead, we just fill in the page array with the kernel virtual
2536 * addresses of the buffers.
2539 MmBuildMdlForNonPagedPool(m)
2542 vm_offset_t *mdl_pages;
2545 pagecnt = SPAN_PAGES(m->mdl_byteoffset, m->mdl_bytecount);
2547 if (pagecnt > (m->mdl_size - sizeof(mdl)) / sizeof(vm_offset_t *))
2548 panic("not enough pages in MDL to describe buffer");
2550 mdl_pages = MmGetMdlPfnArray(m);
2552 for (i = 0; i < pagecnt; i++)
2553 *mdl_pages = (vm_offset_t)m->mdl_startva + (i * PAGE_SIZE);
2555 m->mdl_flags |= MDL_SOURCE_IS_NONPAGED_POOL;
2556 m->mdl_mappedsystemva = MmGetMdlVirtualAddress(m);
2560 MmMapLockedPages(mdl *buf, uint8_t accessmode)
2562 buf->mdl_flags |= MDL_MAPPED_TO_SYSTEM_VA;
2563 return (MmGetMdlVirtualAddress(buf));
2567 MmMapLockedPagesSpecifyCache(mdl *buf, uint8_t accessmode, uint32_t cachetype,
2568 void *vaddr, uint32_t bugcheck, uint32_t prio)
2570 return (MmMapLockedPages(buf, accessmode));
2574 MmUnmapLockedPages(vaddr, buf)
2578 buf->mdl_flags &= ~MDL_MAPPED_TO_SYSTEM_VA;
2582 * This function has a problem in that it will break if you
2583 * compile this module without PAE and try to use it on a PAE
2584 * kernel. Unfortunately, there's no way around this at the
2585 * moment. It's slightly less broken that using pmap_kextract().
2586 * You'd think the virtual memory subsystem would help us out
2587 * here, but it doesn't.
2591 MmGetPhysicalAddress(void *base)
2593 return (pmap_extract(kernel_map->pmap, (vm_offset_t)base));
2597 MmGetSystemRoutineAddress(ustr)
2598 unicode_string *ustr;
2602 if (RtlUnicodeStringToAnsiString(&astr, ustr, TRUE))
2604 return (ndis_get_routine_address(ntoskrnl_functbl, astr.as_buf));
2608 MmIsAddressValid(vaddr)
2611 if (pmap_extract(kernel_map->pmap, (vm_offset_t)vaddr))
2618 MmMapIoSpace(paddr, len, cachetype)
2623 devclass_t nexus_class;
2624 device_t *nexus_devs, devp;
2625 int nexus_count = 0;
2626 device_t matching_dev = NULL;
2627 struct resource *res;
2631 /* There will always be at least one nexus. */
2633 nexus_class = devclass_find("nexus");
2634 devclass_get_devices(nexus_class, &nexus_devs, &nexus_count);
2636 for (i = 0; i < nexus_count; i++) {
2637 devp = nexus_devs[i];
2638 matching_dev = ntoskrnl_finddev(devp, paddr, &res);
2643 free(nexus_devs, M_TEMP);
2645 if (matching_dev == NULL)
2648 v = (vm_offset_t)rman_get_virtual(res);
2649 if (paddr > rman_get_start(res))
2650 v += paddr - rman_get_start(res);
2656 MmUnmapIoSpace(vaddr, len)
2664 ntoskrnl_finddev(dev, paddr, res)
2667 struct resource **res;
2669 device_t *children = NULL;
2670 device_t matching_dev;
2673 struct resource_list *rl;
2674 struct resource_list_entry *rle;
2678 /* We only want devices that have been successfully probed. */
2680 if (device_is_alive(dev) == FALSE)
2683 rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
2685 STAILQ_FOREACH(rle, rl, link) {
2691 flags = rman_get_flags(r);
2693 if (rle->type == SYS_RES_MEMORY &&
2694 paddr >= rman_get_start(r) &&
2695 paddr <= rman_get_end(r)) {
2696 if (!(flags & RF_ACTIVE))
2697 bus_activate_resource(dev,
2698 SYS_RES_MEMORY, 0, r);
2706 * If this device has children, do another
2707 * level of recursion to inspect them.
2710 device_get_children(dev, &children, &childcnt);
2712 for (i = 0; i < childcnt; i++) {
2713 matching_dev = ntoskrnl_finddev(children[i], paddr, res);
2714 if (matching_dev != NULL) {
2715 free(children, M_TEMP);
2716 return (matching_dev);
2721 /* Won't somebody please think of the children! */
2723 if (children != NULL)
2724 free(children, M_TEMP);
2730 * Workitems are unlike DPCs, in that they run in a user-mode thread
2731 * context rather than at DISPATCH_LEVEL in kernel context. In our
2732 * case we run them in kernel context anyway.
2735 ntoskrnl_workitem_thread(arg)
2745 InitializeListHead(&kq->kq_disp);
2746 kq->kq_td = curthread;
2748 KeInitializeSpinLock(&kq->kq_lock);
2749 KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
2752 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
2754 KeAcquireSpinLock(&kq->kq_lock, &irql);
2758 KeReleaseSpinLock(&kq->kq_lock, irql);
2762 while (!IsListEmpty(&kq->kq_disp)) {
2763 l = RemoveHeadList(&kq->kq_disp);
2764 iw = CONTAINING_RECORD(l,
2765 io_workitem, iw_listentry);
2766 InitializeListHead((&iw->iw_listentry));
2767 if (iw->iw_func == NULL)
2769 KeReleaseSpinLock(&kq->kq_lock, irql);
2770 MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx);
2771 KeAcquireSpinLock(&kq->kq_lock, &irql);
2774 KeReleaseSpinLock(&kq->kq_lock, irql);
2778 return; /* notreached */
2782 RtlCharToInteger(src, base, val)
2791 return (STATUS_ACCESS_VIOLATION);
2792 while (*src != '\0' && *src <= ' ')
2796 else if (*src == '-') {
2807 } else if (*src == 'o') {
2810 } else if (*src == 'x') {
2815 } else if (!(base == 2 || base == 8 || base == 10 || base == 16))
2816 return (STATUS_INVALID_PARAMETER);
2818 for (res = 0; *src; src++) {
2822 else if (isxdigit(*src))
2823 v = tolower(*src) - 'a' + 10;
2827 return (STATUS_INVALID_PARAMETER);
2828 res = res * base + v;
2830 *val = negative ? -res : res;
2831 return (STATUS_SUCCESS);
2835 ntoskrnl_destroy_workitem_threads(void)
2840 for (i = 0; i < WORKITEM_THREADS; i++) {
2843 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2845 tsleep(kq->kq_td->td_proc, PWAIT, "waitiw", hz/10);
2850 IoAllocateWorkItem(dobj)
2851 device_object *dobj;
2855 iw = uma_zalloc(iw_zone, M_NOWAIT);
2859 InitializeListHead(&iw->iw_listentry);
2862 mtx_lock(&ntoskrnl_dispatchlock);
2863 iw->iw_idx = wq_idx;
2864 WORKIDX_INC(wq_idx);
2865 mtx_unlock(&ntoskrnl_dispatchlock);
2874 uma_zfree(iw_zone, iw);
2878 IoQueueWorkItem(iw, iw_func, qtype, ctx)
2880 io_workitem_func iw_func;
2889 kq = wq_queues + iw->iw_idx;
2891 KeAcquireSpinLock(&kq->kq_lock, &irql);
2894 * Traverse the list and make sure this workitem hasn't
2895 * already been inserted. Queuing the same workitem
2896 * twice will hose the list but good.
2899 l = kq->kq_disp.nle_flink;
2900 while (l != &kq->kq_disp) {
2901 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2903 /* Already queued -- do nothing. */
2904 KeReleaseSpinLock(&kq->kq_lock, irql);
2910 iw->iw_func = iw_func;
2913 InsertTailList((&kq->kq_disp), (&iw->iw_listentry));
2914 KeReleaseSpinLock(&kq->kq_lock, irql);
2916 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
2920 ntoskrnl_workitem(dobj, arg)
2921 device_object *dobj;
2929 w = (work_queue_item *)dobj;
2930 f = (work_item_func)w->wqi_func;
2931 uma_zfree(iw_zone, iw);
2932 MSCALL2(f, w, w->wqi_ctx);
2936 * The ExQueueWorkItem() API is deprecated in Windows XP. Microsoft
2937 * warns that it's unsafe and to use IoQueueWorkItem() instead. The
2938 * problem with ExQueueWorkItem() is that it can't guard against
2939 * the condition where a driver submits a job to the work queue and
2940 * is then unloaded before the job is able to run. IoQueueWorkItem()
2941 * acquires a reference to the device's device_object via the
2942 * object manager and retains it until after the job has completed,
2943 * which prevents the driver from being unloaded before the job
2944 * runs. (We don't currently support this behavior, though hopefully
2945 * that will change once the object manager API is fleshed out a bit.)
2947 * Having said all that, the ExQueueWorkItem() API remains, because
2948 * there are still other parts of Windows that use it, including
2949 * NDIS itself: NdisScheduleWorkItem() calls ExQueueWorkItem().
2950 * We fake up the ExQueueWorkItem() API on top of our implementation
2951 * of IoQueueWorkItem(). Workitem thread #3 is reserved exclusively
2952 * for ExQueueWorkItem() jobs, and we pass a pointer to the work
2953 * queue item (provided by the caller) in to IoAllocateWorkItem()
2954 * instead of the device_object. We need to save this pointer so
2955 * we can apply a sanity check: as with the DPC queue and other
2956 * workitem queues, we can't allow the same work queue item to
2957 * be queued twice. If it's already pending, we silently return
2961 ExQueueWorkItem(w, qtype)
2966 io_workitem_func iwf;
2974 * We need to do a special sanity test to make sure
2975 * the ExQueueWorkItem() API isn't used to queue
2976 * the same workitem twice. Rather than checking the
2977 * io_workitem pointer itself, we test the attached
2978 * device object, which is really a pointer to the
2979 * legacy work queue item structure.
2982 kq = wq_queues + WORKITEM_LEGACY_THREAD;
2983 KeAcquireSpinLock(&kq->kq_lock, &irql);
2984 l = kq->kq_disp.nle_flink;
2985 while (l != &kq->kq_disp) {
2986 cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
2987 if (cur->iw_dobj == (device_object *)w) {
2988 /* Already queued -- do nothing. */
2989 KeReleaseSpinLock(&kq->kq_lock, irql);
2994 KeReleaseSpinLock(&kq->kq_lock, irql);
2996 iw = IoAllocateWorkItem((device_object *)w);
3000 iw->iw_idx = WORKITEM_LEGACY_THREAD;
3001 iwf = (io_workitem_func)ntoskrnl_findwrap((funcptr)ntoskrnl_workitem);
3002 IoQueueWorkItem(iw, iwf, qtype, iw);
3006 RtlZeroMemory(dst, len)
3014 RtlSecureZeroMemory(dst, len)
3018 memset(dst, 0, len);
3022 RtlFillMemory(void *dst, size_t len, uint8_t c)
3024 memset(dst, c, len);
3028 RtlMoveMemory(dst, src, len)
3033 memmove(dst, src, len);
3037 RtlCopyMemory(dst, src, len)
3042 bcopy(src, dst, len);
3046 RtlCompareMemory(s1, s2, len)
3054 m1 = __DECONST(char *, s1);
3055 m2 = __DECONST(char *, s2);
3057 for (i = 0; i < len && m1[i] == m2[i]; i++);
3062 RtlInitAnsiString(dst, src)
3072 a->as_len = a->as_maxlen = 0;
3076 a->as_len = a->as_maxlen = strlen(src);
3081 RtlInitUnicodeString(dst, src)
3082 unicode_string *dst;
3092 u->us_len = u->us_maxlen = 0;
3099 u->us_len = u->us_maxlen = i * 2;
3104 RtlUnicodeStringToInteger(ustr, base, val)
3105 unicode_string *ustr;
3114 uchr = ustr->us_buf;
3116 bzero(abuf, sizeof(abuf));
3118 if ((char)((*uchr) & 0xFF) == '-') {
3122 } else if ((char)((*uchr) & 0xFF) == '+') {
3129 if ((char)((*uchr) & 0xFF) == 'b') {
3133 } else if ((char)((*uchr) & 0xFF) == 'o') {
3137 } else if ((char)((*uchr) & 0xFF) == 'x') {
3151 ntoskrnl_unicode_to_ascii(uchr, astr, len);
3152 *val = strtoul(abuf, NULL, base);
3154 return (STATUS_SUCCESS);
3158 RtlFreeUnicodeString(ustr)
3159 unicode_string *ustr;
3161 if (ustr->us_buf == NULL)
3163 ExFreePool(ustr->us_buf);
3164 ustr->us_buf = NULL;
3168 RtlFreeAnsiString(astr)
3171 if (astr->as_buf == NULL)
3173 ExFreePool(astr->as_buf);
3174 astr->as_buf = NULL;
3181 return (int)strtol(str, (char **)NULL, 10);
3188 return strtol(str, (char **)NULL, 10);
3199 srand(unsigned int seed __unused)
3204 IoIsWdmVersionAvailable(uint8_t major, uint8_t minor)
3206 if (major == WDM_MAJOR && minor == WDM_MINOR_WINXP)
3212 IoOpenDeviceRegistryKey(struct device_object *devobj, uint32_t type,
3213 uint32_t mask, void **key)
3215 return (NDIS_STATUS_INVALID_DEVICE_REQUEST);
3219 IoGetDeviceObjectPointer(name, reqaccess, fileobj, devobj)
3220 unicode_string *name;
3223 device_object *devobj;
3225 return (STATUS_SUCCESS);
3229 IoGetDeviceProperty(devobj, regprop, buflen, prop, reslen)
3230 device_object *devobj;
3239 drv = devobj->do_drvobj;
3242 case DEVPROP_DRIVER_KEYNAME:
3244 *name = drv->dro_drivername.us_buf;
3245 *reslen = drv->dro_drivername.us_len;
3248 return (STATUS_INVALID_PARAMETER_2);
3252 return (STATUS_SUCCESS);
3256 KeInitializeMutex(kmutex, level)
3260 InitializeListHead((&kmutex->km_header.dh_waitlisthead));
3261 kmutex->km_abandoned = FALSE;
3262 kmutex->km_apcdisable = 1;
3263 kmutex->km_header.dh_sigstate = 1;
3264 kmutex->km_header.dh_type = DISP_TYPE_MUTANT;
3265 kmutex->km_header.dh_size = sizeof(kmutant) / sizeof(uint32_t);
3266 kmutex->km_ownerthread = NULL;
3270 KeReleaseMutex(kmutant *kmutex, uint8_t kwait)
3274 mtx_lock(&ntoskrnl_dispatchlock);
3275 prevstate = kmutex->km_header.dh_sigstate;
3276 if (kmutex->km_ownerthread != curthread) {
3277 mtx_unlock(&ntoskrnl_dispatchlock);
3278 return (STATUS_MUTANT_NOT_OWNED);
3281 kmutex->km_header.dh_sigstate++;
3282 kmutex->km_abandoned = FALSE;
3284 if (kmutex->km_header.dh_sigstate == 1) {
3285 kmutex->km_ownerthread = NULL;
3286 ntoskrnl_waittest(&kmutex->km_header, IO_NO_INCREMENT);
3289 mtx_unlock(&ntoskrnl_dispatchlock);
3295 KeReadStateMutex(kmutex)
3298 return (kmutex->km_header.dh_sigstate);
3302 KeInitializeEvent(nt_kevent *kevent, uint32_t type, uint8_t state)
3304 InitializeListHead((&kevent->k_header.dh_waitlisthead));
3305 kevent->k_header.dh_sigstate = state;
3306 if (type == EVENT_TYPE_NOTIFY)
3307 kevent->k_header.dh_type = DISP_TYPE_NOTIFICATION_EVENT;
3309 kevent->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_EVENT;
3310 kevent->k_header.dh_size = sizeof(nt_kevent) / sizeof(uint32_t);
3314 KeResetEvent(kevent)
3319 mtx_lock(&ntoskrnl_dispatchlock);
3320 prevstate = kevent->k_header.dh_sigstate;
3321 kevent->k_header.dh_sigstate = FALSE;
3322 mtx_unlock(&ntoskrnl_dispatchlock);
3328 KeSetEvent(nt_kevent *kevent, uint32_t increment, uint8_t kwait)
3332 nt_dispatch_header *dh;
3336 mtx_lock(&ntoskrnl_dispatchlock);
3337 prevstate = kevent->k_header.dh_sigstate;
3338 dh = &kevent->k_header;
3340 if (IsListEmpty(&dh->dh_waitlisthead))
3342 * If there's nobody in the waitlist, just set
3343 * the state to signalled.
3345 dh->dh_sigstate = 1;
3348 * Get the first waiter. If this is a synchronization
3349 * event, just wake up that one thread (don't bother
3350 * setting the state to signalled since we're supposed
3351 * to automatically clear synchronization events anyway).
3353 * If it's a notification event, or the first
3354 * waiter is doing a WAITTYPE_ALL wait, go through
3355 * the full wait satisfaction process.
3357 w = CONTAINING_RECORD(dh->dh_waitlisthead.nle_flink,
3358 wait_block, wb_waitlist);
3361 if (kevent->k_header.dh_type == DISP_TYPE_NOTIFICATION_EVENT ||
3362 w->wb_waittype == WAITTYPE_ALL) {
3363 if (prevstate == 0) {
3364 dh->dh_sigstate = 1;
3365 ntoskrnl_waittest(dh, increment);
3368 w->wb_awakened |= TRUE;
3369 cv_broadcastpri(&we->we_cv,
3370 (w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
3371 w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
3375 mtx_unlock(&ntoskrnl_dispatchlock);
3381 KeClearEvent(kevent)
3384 kevent->k_header.dh_sigstate = FALSE;
3388 KeReadStateEvent(kevent)
3391 return (kevent->k_header.dh_sigstate);
3395 * The object manager in Windows is responsible for managing
3396 * references and access to various types of objects, including
3397 * device_objects, events, threads, timers and so on. However,
3398 * there's a difference in the way objects are handled in user
3399 * mode versus kernel mode.
3401 * In user mode (i.e. Win32 applications), all objects are
3402 * managed by the object manager. For example, when you create
3403 * a timer or event object, you actually end up with an
3404 * object_header (for the object manager's bookkeeping
3405 * purposes) and an object body (which contains the actual object
3406 * structure, e.g. ktimer, kevent, etc...). This allows Windows
3407 * to manage resource quotas and to enforce access restrictions
3408 * on basically every kind of system object handled by the kernel.
3410 * However, in kernel mode, you only end up using the object
3411 * manager some of the time. For example, in a driver, you create
3412 * a timer object by simply allocating the memory for a ktimer
3413 * structure and initializing it with KeInitializeTimer(). Hence,
3414 * the timer has no object_header and no reference counting or
3415 * security/resource checks are done on it. The assumption in
3416 * this case is that if you're running in kernel mode, you know
3417 * what you're doing, and you're already at an elevated privilege
3420 * There are some exceptions to this. The two most important ones
3421 * for our purposes are device_objects and threads. We need to use
3422 * the object manager to do reference counting on device_objects,
3423 * and for threads, you can only get a pointer to a thread's
3424 * dispatch header by using ObReferenceObjectByHandle() on the
3425 * handle returned by PsCreateSystemThread().
3429 ObReferenceObjectByHandle(ndis_handle handle, uint32_t reqaccess, void *otype,
3430 uint8_t accessmode, void **object, void **handleinfo)
3434 nr = malloc(sizeof(nt_objref), M_DEVBUF, M_NOWAIT|M_ZERO);
3436 return (STATUS_INSUFFICIENT_RESOURCES);
3438 InitializeListHead((&nr->no_dh.dh_waitlisthead));
3439 nr->no_obj = handle;
3440 nr->no_dh.dh_type = DISP_TYPE_THREAD;
3441 nr->no_dh.dh_sigstate = 0;
3442 nr->no_dh.dh_size = (uint8_t)(sizeof(struct thread) /
3444 TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link);
3447 return (STATUS_SUCCESS);
3451 ObfDereferenceObject(object)
3457 TAILQ_REMOVE(&ntoskrnl_reflist, nr, link);
3465 return (STATUS_SUCCESS);
3469 WmiQueryTraceInformation(traceclass, traceinfo, infolen, reqlen, buf)
3470 uint32_t traceclass;
3476 return (STATUS_NOT_FOUND);
3480 WmiTraceMessage(uint64_t loghandle, uint32_t messageflags,
3481 void *guid, uint16_t messagenum, ...)
3483 return (STATUS_SUCCESS);
3487 IoWMIRegistrationControl(dobj, action)
3488 device_object *dobj;
3491 return (STATUS_SUCCESS);
3495 * This is here just in case the thread returns without calling
3496 * PsTerminateSystemThread().
3499 ntoskrnl_thrfunc(arg)
3502 thread_context *thrctx;
3503 uint32_t (*tfunc)(void *);
3508 tfunc = thrctx->tc_thrfunc;
3509 tctx = thrctx->tc_thrctx;
3510 free(thrctx, M_TEMP);
3512 rval = MSCALL1(tfunc, tctx);
3514 PsTerminateSystemThread(rval);
3515 return; /* notreached */
3519 PsCreateSystemThread(handle, reqaccess, objattrs, phandle,
3520 clientid, thrfunc, thrctx)
3521 ndis_handle *handle;
3524 ndis_handle phandle;
3533 tc = malloc(sizeof(thread_context), M_TEMP, M_NOWAIT);
3535 return (STATUS_INSUFFICIENT_RESOURCES);
3537 tc->tc_thrctx = thrctx;
3538 tc->tc_thrfunc = thrfunc;
3540 error = kproc_create(ntoskrnl_thrfunc, tc, &p,
3541 RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Kthread %d", ntoskrnl_kth);
3545 return (STATUS_INSUFFICIENT_RESOURCES);
3551 return (STATUS_SUCCESS);
3555 * In Windows, the exit of a thread is an event that you're allowed
3556 * to wait on, assuming you've obtained a reference to the thread using
3557 * ObReferenceObjectByHandle(). Unfortunately, the only way we can
3558 * simulate this behavior is to register each thread we create in a
3559 * reference list, and if someone holds a reference to us, we poke
3563 PsTerminateSystemThread(status)
3566 struct nt_objref *nr;
3568 mtx_lock(&ntoskrnl_dispatchlock);
3569 TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) {
3570 if (nr->no_obj != curthread->td_proc)
3572 nr->no_dh.dh_sigstate = 1;
3573 ntoskrnl_waittest(&nr->no_dh, IO_NO_INCREMENT);
3576 mtx_unlock(&ntoskrnl_dispatchlock);
3581 return (0); /* notreached */
3585 DbgPrint(char *fmt, ...)
3595 return (STATUS_SUCCESS);
3602 kdb_enter(KDB_WHY_NDIS, "DbgBreakPoint(): breakpoint");
3606 KeBugCheckEx(code, param1, param2, param3, param4)
3613 panic("KeBugCheckEx: STOP 0x%X", code);
3617 ntoskrnl_timercall(arg)
3624 mtx_lock(&ntoskrnl_dispatchlock);
3628 #ifdef NTOSKRNL_DEBUG_TIMERS
3629 ntoskrnl_timer_fires++;
3631 ntoskrnl_remove_timer(timer);
3634 * This should never happen, but complain
3638 if (timer->k_header.dh_inserted == FALSE) {
3639 mtx_unlock(&ntoskrnl_dispatchlock);
3640 printf("NTOS: timer %p fired even though "
3641 "it was canceled\n", timer);
3645 /* Mark the timer as no longer being on the timer queue. */
3647 timer->k_header.dh_inserted = FALSE;
3649 /* Now signal the object and satisfy any waits on it. */
3651 timer->k_header.dh_sigstate = 1;
3652 ntoskrnl_waittest(&timer->k_header, IO_NO_INCREMENT);
3655 * If this is a periodic timer, re-arm it
3656 * so it will fire again. We do this before
3657 * calling any deferred procedure calls because
3658 * it's possible the DPC might cancel the timer,
3659 * in which case it would be wrong for us to
3660 * re-arm it again afterwards.
3663 if (timer->k_period) {
3665 tv.tv_usec = timer->k_period * 1000;
3666 timer->k_header.dh_inserted = TRUE;
3667 ntoskrnl_insert_timer(timer, tvtohz(&tv));
3668 #ifdef NTOSKRNL_DEBUG_TIMERS
3669 ntoskrnl_timer_reloads++;
3675 mtx_unlock(&ntoskrnl_dispatchlock);
3677 /* If there's a DPC associated with the timer, queue it up. */
3680 KeInsertQueueDpc(dpc, NULL, NULL);
3683 #ifdef NTOSKRNL_DEBUG_TIMERS
3685 sysctl_show_timers(SYSCTL_HANDLER_ARGS)
3690 ntoskrnl_show_timers();
3691 return (sysctl_handle_int(oidp, &ret, 0, req));
3695 ntoskrnl_show_timers()
3700 mtx_lock_spin(&ntoskrnl_calllock);
3701 l = ntoskrnl_calllist.nle_flink;
3702 while(l != &ntoskrnl_calllist) {
3706 mtx_unlock_spin(&ntoskrnl_calllock);
3709 printf("%d timers available (out of %d)\n", i, NTOSKRNL_TIMEOUTS);
3710 printf("timer sets: %qu\n", ntoskrnl_timer_sets);
3711 printf("timer reloads: %qu\n", ntoskrnl_timer_reloads);
3712 printf("timer cancels: %qu\n", ntoskrnl_timer_cancels);
3713 printf("timer fires: %qu\n", ntoskrnl_timer_fires);
3719 * Must be called with dispatcher lock held.
3723 ntoskrnl_insert_timer(timer, ticks)
3732 * Try and allocate a timer.
3734 mtx_lock_spin(&ntoskrnl_calllock);
3735 if (IsListEmpty(&ntoskrnl_calllist)) {
3736 mtx_unlock_spin(&ntoskrnl_calllock);
3737 #ifdef NTOSKRNL_DEBUG_TIMERS
3738 ntoskrnl_show_timers();
3740 panic("out of timers!");
3742 l = RemoveHeadList(&ntoskrnl_calllist);
3743 mtx_unlock_spin(&ntoskrnl_calllock);
3745 e = CONTAINING_RECORD(l, callout_entry, ce_list);
3748 timer->k_callout = c;
3751 callout_reset(c, ticks, ntoskrnl_timercall, timer);
3755 ntoskrnl_remove_timer(timer)
3760 e = (callout_entry *)timer->k_callout;
3761 callout_stop(timer->k_callout);
3763 mtx_lock_spin(&ntoskrnl_calllock);
3764 InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
3765 mtx_unlock_spin(&ntoskrnl_calllock);
3769 KeInitializeTimer(timer)
3775 KeInitializeTimerEx(timer, EVENT_TYPE_NOTIFY);
3779 KeInitializeTimerEx(timer, type)
3786 bzero((char *)timer, sizeof(ktimer));
3787 InitializeListHead((&timer->k_header.dh_waitlisthead));
3788 timer->k_header.dh_sigstate = FALSE;
3789 timer->k_header.dh_inserted = FALSE;
3790 if (type == EVENT_TYPE_NOTIFY)
3791 timer->k_header.dh_type = DISP_TYPE_NOTIFICATION_TIMER;
3793 timer->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_TIMER;
3794 timer->k_header.dh_size = sizeof(ktimer) / sizeof(uint32_t);
3798 * DPC subsystem. A Windows Defered Procedure Call has the following
3800 * - It runs at DISPATCH_LEVEL.
3801 * - It can have one of 3 importance values that control when it
3802 * runs relative to other DPCs in the queue.
3803 * - On SMP systems, it can be set to run on a specific processor.
3804 * In order to satisfy the last property, we create a DPC thread for
3805 * each CPU in the system and bind it to that CPU. Each thread
3806 * maintains three queues with different importance levels, which
3807 * will be processed in order from lowest to highest.
3809 * In Windows, interrupt handlers run as DPCs. (Not to be confused
3810 * with ISRs, which run in interrupt context and can preempt DPCs.)
3811 * ISRs are given the highest importance so that they'll take
3812 * precedence over timers and other things.
3816 ntoskrnl_dpc_thread(arg)
3826 InitializeListHead(&kq->kq_disp);
3827 kq->kq_td = curthread;
3829 kq->kq_running = FALSE;
3830 KeInitializeSpinLock(&kq->kq_lock);
3831 KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
3832 KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE);
3835 * Elevate our priority. DPCs are used to run interrupt
3836 * handlers, and they should trigger as soon as possible
3837 * once scheduled by an ISR.
3840 thread_lock(curthread);
3841 #ifdef NTOSKRNL_MULTIPLE_DPCS
3842 sched_bind(curthread, kq->kq_cpu);
3844 sched_prio(curthread, PRI_MIN_KERN);
3845 thread_unlock(curthread);
3848 KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
3850 KeAcquireSpinLock(&kq->kq_lock, &irql);
3854 KeReleaseSpinLock(&kq->kq_lock, irql);
3858 kq->kq_running = TRUE;
3860 while (!IsListEmpty(&kq->kq_disp)) {
3861 l = RemoveHeadList((&kq->kq_disp));
3862 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3863 InitializeListHead((&d->k_dpclistentry));
3864 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
3865 MSCALL4(d->k_deferedfunc, d, d->k_deferredctx,
3866 d->k_sysarg1, d->k_sysarg2);
3867 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3870 kq->kq_running = FALSE;
3872 KeReleaseSpinLock(&kq->kq_lock, irql);
3874 KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE);
3878 return; /* notreached */
3882 ntoskrnl_destroy_dpc_threads(void)
3889 #ifdef NTOSKRNL_MULTIPLE_DPCS
3890 for (i = 0; i < mp_ncpus; i++) {
3892 for (i = 0; i < 1; i++) {
3897 KeInitializeDpc(&dpc, NULL, NULL);
3898 KeSetTargetProcessorDpc(&dpc, i);
3899 KeInsertQueueDpc(&dpc, NULL, NULL);
3901 tsleep(kq->kq_td->td_proc, PWAIT, "dpcw", hz/10);
3906 ntoskrnl_insert_dpc(head, dpc)
3913 l = head->nle_flink;
3915 d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
3921 if (dpc->k_importance == KDPC_IMPORTANCE_LOW)
3922 InsertTailList((head), (&dpc->k_dpclistentry));
3924 InsertHeadList((head), (&dpc->k_dpclistentry));
3930 KeInitializeDpc(dpc, dpcfunc, dpcctx)
3939 dpc->k_deferedfunc = dpcfunc;
3940 dpc->k_deferredctx = dpcctx;
3941 dpc->k_num = KDPC_CPU_DEFAULT;
3942 dpc->k_importance = KDPC_IMPORTANCE_MEDIUM;
3943 InitializeListHead((&dpc->k_dpclistentry));
3947 KeInsertQueueDpc(dpc, sysarg1, sysarg2)
3961 #ifdef NTOSKRNL_MULTIPLE_DPCS
3962 KeRaiseIrql(DISPATCH_LEVEL, &irql);
3965 * By default, the DPC is queued to run on the same CPU
3966 * that scheduled it.
3969 if (dpc->k_num == KDPC_CPU_DEFAULT)
3970 kq += curthread->td_oncpu;
3973 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
3975 KeAcquireSpinLock(&kq->kq_lock, &irql);
3978 r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc);
3980 dpc->k_sysarg1 = sysarg1;
3981 dpc->k_sysarg2 = sysarg2;
3983 KeReleaseSpinLock(&kq->kq_lock, irql);
3988 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
3994 KeRemoveQueueDpc(dpc)
4003 #ifdef NTOSKRNL_MULTIPLE_DPCS
4004 KeRaiseIrql(DISPATCH_LEVEL, &irql);
4006 kq = kq_queues + dpc->k_num;
4008 KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
4011 KeAcquireSpinLock(&kq->kq_lock, &irql);
4014 if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) {
4015 KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
4020 RemoveEntryList((&dpc->k_dpclistentry));
4021 InitializeListHead((&dpc->k_dpclistentry));
4023 KeReleaseSpinLock(&kq->kq_lock, irql);
4029 KeSetImportanceDpc(dpc, imp)
4033 if (imp != KDPC_IMPORTANCE_LOW &&
4034 imp != KDPC_IMPORTANCE_MEDIUM &&
4035 imp != KDPC_IMPORTANCE_HIGH)
4038 dpc->k_importance = (uint8_t)imp;
4042 KeSetTargetProcessorDpc(kdpc *dpc, uint8_t cpu)
4051 KeFlushQueuedDpcs(void)
4057 * Poke each DPC queue and wait
4058 * for them to drain.
4061 #ifdef NTOSKRNL_MULTIPLE_DPCS
4062 for (i = 0; i < mp_ncpus; i++) {
4064 for (i = 0; i < 1; i++) {
4067 KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
4068 KeWaitForSingleObject(&kq->kq_done, 0, 0, TRUE, NULL);
4073 KeGetCurrentProcessorNumber(void)
4075 return ((uint32_t)curthread->td_oncpu);
4079 KeSetTimerEx(timer, duetime, period, dpc)
4092 mtx_lock(&ntoskrnl_dispatchlock);
4094 if (timer->k_header.dh_inserted == TRUE) {
4095 ntoskrnl_remove_timer(timer);
4096 #ifdef NTOSKRNL_DEBUG_TIMERS
4097 ntoskrnl_timer_cancels++;
4099 timer->k_header.dh_inserted = FALSE;
4104 timer->k_duetime = duetime;
4105 timer->k_period = period;
4106 timer->k_header.dh_sigstate = FALSE;
4110 tv.tv_sec = - (duetime) / 10000000;
4111 tv.tv_usec = (- (duetime) / 10) -
4112 (tv.tv_sec * 1000000);
4114 ntoskrnl_time(&curtime);
4115 if (duetime < curtime)
4116 tv.tv_sec = tv.tv_usec = 0;
4118 tv.tv_sec = ((duetime) - curtime) / 10000000;
4119 tv.tv_usec = ((duetime) - curtime) / 10 -
4120 (tv.tv_sec * 1000000);
4124 timer->k_header.dh_inserted = TRUE;
4125 ntoskrnl_insert_timer(timer, tvtohz(&tv));
4126 #ifdef NTOSKRNL_DEBUG_TIMERS
4127 ntoskrnl_timer_sets++;
4130 mtx_unlock(&ntoskrnl_dispatchlock);
4136 KeSetTimer(timer, duetime, dpc)
4141 return (KeSetTimerEx(timer, duetime, 0, dpc));
4145 * The Windows DDK documentation seems to say that cancelling
4146 * a timer that has a DPC will result in the DPC also being
4147 * cancelled, but this isn't really the case.
4151 KeCancelTimer(timer)
4159 mtx_lock(&ntoskrnl_dispatchlock);
4161 pending = timer->k_header.dh_inserted;
4163 if (timer->k_header.dh_inserted == TRUE) {
4164 timer->k_header.dh_inserted = FALSE;
4165 ntoskrnl_remove_timer(timer);
4166 #ifdef NTOSKRNL_DEBUG_TIMERS
4167 ntoskrnl_timer_cancels++;
4171 mtx_unlock(&ntoskrnl_dispatchlock);
4177 KeReadStateTimer(timer)
4180 return (timer->k_header.dh_sigstate);
4184 KeDelayExecutionThread(uint8_t wait_mode, uint8_t alertable, int64_t *interval)
4189 panic("invalid wait_mode %d", wait_mode);
4191 KeInitializeTimer(&timer);
4192 KeSetTimer(&timer, *interval, NULL);
4193 KeWaitForSingleObject(&timer, 0, 0, alertable, NULL);
4195 return STATUS_SUCCESS;
4199 KeQueryInterruptTime(void)
4204 getmicrouptime(&tv);
4206 ticks = tvtohz(&tv);
4208 return ticks * howmany(10000000, hz);
4211 static struct thread *
4212 KeGetCurrentThread(void)
4219 KeSetPriorityThread(td, pri)
4226 return LOW_REALTIME_PRIORITY;
4228 if (td->td_priority <= PRI_MIN_KERN)
4229 old = HIGH_PRIORITY;
4230 else if (td->td_priority >= PRI_MAX_KERN)
4233 old = LOW_REALTIME_PRIORITY;
4236 if (pri == HIGH_PRIORITY)
4237 sched_prio(td, PRI_MIN_KERN);
4238 if (pri == LOW_REALTIME_PRIORITY)
4239 sched_prio(td, PRI_MIN_KERN + (PRI_MAX_KERN - PRI_MIN_KERN) / 2);
4240 if (pri == LOW_PRIORITY)
4241 sched_prio(td, PRI_MAX_KERN);
4250 printf("ntoskrnl dummy called...\n");
4254 image_patch_table ntoskrnl_functbl[] = {
4255 IMPORT_SFUNC(RtlZeroMemory, 2),
4256 IMPORT_SFUNC(RtlSecureZeroMemory, 2),
4257 IMPORT_SFUNC(RtlFillMemory, 3),
4258 IMPORT_SFUNC(RtlMoveMemory, 3),
4259 IMPORT_SFUNC(RtlCharToInteger, 3),
4260 IMPORT_SFUNC(RtlCopyMemory, 3),
4261 IMPORT_SFUNC(RtlCopyString, 2),
4262 IMPORT_SFUNC(RtlCompareMemory, 3),
4263 IMPORT_SFUNC(RtlEqualUnicodeString, 3),
4264 IMPORT_SFUNC(RtlCopyUnicodeString, 2),
4265 IMPORT_SFUNC(RtlUnicodeStringToAnsiString, 3),
4266 IMPORT_SFUNC(RtlAnsiStringToUnicodeString, 3),
4267 IMPORT_SFUNC(RtlInitAnsiString, 2),
4268 IMPORT_SFUNC_MAP(RtlInitString, RtlInitAnsiString, 2),
4269 IMPORT_SFUNC(RtlInitUnicodeString, 2),
4270 IMPORT_SFUNC(RtlFreeAnsiString, 1),
4271 IMPORT_SFUNC(RtlFreeUnicodeString, 1),
4272 IMPORT_SFUNC(RtlUnicodeStringToInteger, 3),
4273 IMPORT_CFUNC(sprintf, 0),
4274 IMPORT_CFUNC(vsprintf, 0),
4275 IMPORT_CFUNC_MAP(_snprintf, snprintf, 0),
4276 IMPORT_CFUNC_MAP(_vsnprintf, vsnprintf, 0),
4277 IMPORT_CFUNC(DbgPrint, 0),
4278 IMPORT_SFUNC(DbgBreakPoint, 0),
4279 IMPORT_SFUNC(KeBugCheckEx, 5),
4280 IMPORT_CFUNC(strncmp, 0),
4281 IMPORT_CFUNC(strcmp, 0),
4282 IMPORT_CFUNC_MAP(stricmp, strcasecmp, 0),
4283 IMPORT_CFUNC(strncpy, 0),
4284 IMPORT_CFUNC(strcpy, 0),
4285 IMPORT_CFUNC(strlen, 0),
4286 IMPORT_CFUNC_MAP(toupper, ntoskrnl_toupper, 0),
4287 IMPORT_CFUNC_MAP(tolower, ntoskrnl_tolower, 0),
4288 IMPORT_CFUNC_MAP(strstr, ntoskrnl_strstr, 0),
4289 IMPORT_CFUNC_MAP(strncat, ntoskrnl_strncat, 0),
4290 IMPORT_CFUNC_MAP(strchr, index, 0),
4291 IMPORT_CFUNC_MAP(strrchr, rindex, 0),
4292 IMPORT_CFUNC(memcpy, 0),
4293 IMPORT_CFUNC_MAP(memmove, ntoskrnl_memmove, 0),
4294 IMPORT_CFUNC_MAP(memset, ntoskrnl_memset, 0),
4295 IMPORT_CFUNC_MAP(memchr, ntoskrnl_memchr, 0),
4296 IMPORT_SFUNC(IoAllocateDriverObjectExtension, 4),
4297 IMPORT_SFUNC(IoGetDriverObjectExtension, 2),
4298 IMPORT_FFUNC(IofCallDriver, 2),
4299 IMPORT_FFUNC(IofCompleteRequest, 2),
4300 IMPORT_SFUNC(IoAcquireCancelSpinLock, 1),
4301 IMPORT_SFUNC(IoReleaseCancelSpinLock, 1),
4302 IMPORT_SFUNC(IoCancelIrp, 1),
4303 IMPORT_SFUNC(IoConnectInterrupt, 11),
4304 IMPORT_SFUNC(IoDisconnectInterrupt, 1),
4305 IMPORT_SFUNC(IoCreateDevice, 7),
4306 IMPORT_SFUNC(IoDeleteDevice, 1),
4307 IMPORT_SFUNC(IoGetAttachedDevice, 1),
4308 IMPORT_SFUNC(IoAttachDeviceToDeviceStack, 2),
4309 IMPORT_SFUNC(IoDetachDevice, 1),
4310 IMPORT_SFUNC(IoBuildSynchronousFsdRequest, 7),
4311 IMPORT_SFUNC(IoBuildAsynchronousFsdRequest, 6),
4312 IMPORT_SFUNC(IoBuildDeviceIoControlRequest, 9),
4313 IMPORT_SFUNC(IoAllocateIrp, 2),
4314 IMPORT_SFUNC(IoReuseIrp, 2),
4315 IMPORT_SFUNC(IoMakeAssociatedIrp, 2),
4316 IMPORT_SFUNC(IoFreeIrp, 1),
4317 IMPORT_SFUNC(IoInitializeIrp, 3),
4318 IMPORT_SFUNC(KeAcquireInterruptSpinLock, 1),
4319 IMPORT_SFUNC(KeReleaseInterruptSpinLock, 2),
4320 IMPORT_SFUNC(KeSynchronizeExecution, 3),
4321 IMPORT_SFUNC(KeWaitForSingleObject, 5),
4322 IMPORT_SFUNC(KeWaitForMultipleObjects, 8),
4323 IMPORT_SFUNC(_allmul, 4),
4324 IMPORT_SFUNC(_alldiv, 4),
4325 IMPORT_SFUNC(_allrem, 4),
4326 IMPORT_RFUNC(_allshr, 0),
4327 IMPORT_RFUNC(_allshl, 0),
4328 IMPORT_SFUNC(_aullmul, 4),
4329 IMPORT_SFUNC(_aulldiv, 4),
4330 IMPORT_SFUNC(_aullrem, 4),
4331 IMPORT_RFUNC(_aullshr, 0),
4332 IMPORT_RFUNC(_aullshl, 0),
4333 IMPORT_CFUNC(atoi, 0),
4334 IMPORT_CFUNC(atol, 0),
4335 IMPORT_CFUNC(rand, 0),
4336 IMPORT_CFUNC(srand, 0),
4337 IMPORT_SFUNC(WRITE_REGISTER_USHORT, 2),
4338 IMPORT_SFUNC(READ_REGISTER_USHORT, 1),
4339 IMPORT_SFUNC(WRITE_REGISTER_ULONG, 2),
4340 IMPORT_SFUNC(READ_REGISTER_ULONG, 1),
4341 IMPORT_SFUNC(READ_REGISTER_UCHAR, 1),
4342 IMPORT_SFUNC(WRITE_REGISTER_UCHAR, 2),
4343 IMPORT_SFUNC(ExInitializePagedLookasideList, 7),
4344 IMPORT_SFUNC(ExDeletePagedLookasideList, 1),
4345 IMPORT_SFUNC(ExInitializeNPagedLookasideList, 7),
4346 IMPORT_SFUNC(ExDeleteNPagedLookasideList, 1),
4347 IMPORT_FFUNC(InterlockedPopEntrySList, 1),
4348 IMPORT_FFUNC(InitializeSListHead, 1),
4349 IMPORT_FFUNC(InterlockedPushEntrySList, 2),
4350 IMPORT_SFUNC(ExQueryDepthSList, 1),
4351 IMPORT_FFUNC_MAP(ExpInterlockedPopEntrySList,
4352 InterlockedPopEntrySList, 1),
4353 IMPORT_FFUNC_MAP(ExpInterlockedPushEntrySList,
4354 InterlockedPushEntrySList, 2),
4355 IMPORT_FFUNC(ExInterlockedPopEntrySList, 2),
4356 IMPORT_FFUNC(ExInterlockedPushEntrySList, 3),
4357 IMPORT_SFUNC(ExAllocatePoolWithTag, 3),
4358 IMPORT_SFUNC(ExFreePoolWithTag, 2),
4359 IMPORT_SFUNC(ExFreePool, 1),
4361 IMPORT_FFUNC(KefAcquireSpinLockAtDpcLevel, 1),
4362 IMPORT_FFUNC(KefReleaseSpinLockFromDpcLevel,1),
4363 IMPORT_FFUNC(KeAcquireSpinLockRaiseToDpc, 1),
4366 * For AMD64, we can get away with just mapping
4367 * KeAcquireSpinLockRaiseToDpc() directly to KfAcquireSpinLock()
4368 * because the calling conventions end up being the same.
4369 * On i386, we have to be careful because KfAcquireSpinLock()
4370 * is _fastcall but KeAcquireSpinLockRaiseToDpc() isn't.
4372 IMPORT_SFUNC(KeAcquireSpinLockAtDpcLevel, 1),
4373 IMPORT_SFUNC(KeReleaseSpinLockFromDpcLevel, 1),
4374 IMPORT_SFUNC_MAP(KeAcquireSpinLockRaiseToDpc, KfAcquireSpinLock, 1),
4376 IMPORT_SFUNC_MAP(KeReleaseSpinLock, KfReleaseSpinLock, 1),
4377 IMPORT_FFUNC(InterlockedIncrement, 1),
4378 IMPORT_FFUNC(InterlockedDecrement, 1),
4379 IMPORT_FFUNC(InterlockedExchange, 2),
4380 IMPORT_FFUNC(ExInterlockedAddLargeStatistic, 2),
4381 IMPORT_SFUNC(IoAllocateMdl, 5),
4382 IMPORT_SFUNC(IoFreeMdl, 1),
4383 IMPORT_SFUNC(MmAllocateContiguousMemory, 2 + 1),
4384 IMPORT_SFUNC(MmAllocateContiguousMemorySpecifyCache, 5 + 3),
4385 IMPORT_SFUNC(MmFreeContiguousMemory, 1),
4386 IMPORT_SFUNC(MmFreeContiguousMemorySpecifyCache, 3),
4387 IMPORT_SFUNC(MmSizeOfMdl, 1),
4388 IMPORT_SFUNC(MmMapLockedPages, 2),
4389 IMPORT_SFUNC(MmMapLockedPagesSpecifyCache, 6),
4390 IMPORT_SFUNC(MmUnmapLockedPages, 2),
4391 IMPORT_SFUNC(MmBuildMdlForNonPagedPool, 1),
4392 IMPORT_SFUNC(MmGetPhysicalAddress, 1),
4393 IMPORT_SFUNC(MmGetSystemRoutineAddress, 1),
4394 IMPORT_SFUNC(MmIsAddressValid, 1),
4395 IMPORT_SFUNC(MmMapIoSpace, 3 + 1),
4396 IMPORT_SFUNC(MmUnmapIoSpace, 2),
4397 IMPORT_SFUNC(KeInitializeSpinLock, 1),
4398 IMPORT_SFUNC(IoIsWdmVersionAvailable, 2),
4399 IMPORT_SFUNC(IoOpenDeviceRegistryKey, 4),
4400 IMPORT_SFUNC(IoGetDeviceObjectPointer, 4),
4401 IMPORT_SFUNC(IoGetDeviceProperty, 5),
4402 IMPORT_SFUNC(IoAllocateWorkItem, 1),
4403 IMPORT_SFUNC(IoFreeWorkItem, 1),
4404 IMPORT_SFUNC(IoQueueWorkItem, 4),
4405 IMPORT_SFUNC(ExQueueWorkItem, 2),
4406 IMPORT_SFUNC(ntoskrnl_workitem, 2),
4407 IMPORT_SFUNC(KeInitializeMutex, 2),
4408 IMPORT_SFUNC(KeReleaseMutex, 2),
4409 IMPORT_SFUNC(KeReadStateMutex, 1),
4410 IMPORT_SFUNC(KeInitializeEvent, 3),
4411 IMPORT_SFUNC(KeSetEvent, 3),
4412 IMPORT_SFUNC(KeResetEvent, 1),
4413 IMPORT_SFUNC(KeClearEvent, 1),
4414 IMPORT_SFUNC(KeReadStateEvent, 1),
4415 IMPORT_SFUNC(KeInitializeTimer, 1),
4416 IMPORT_SFUNC(KeInitializeTimerEx, 2),
4417 IMPORT_SFUNC(KeSetTimer, 3),
4418 IMPORT_SFUNC(KeSetTimerEx, 4),
4419 IMPORT_SFUNC(KeCancelTimer, 1),
4420 IMPORT_SFUNC(KeReadStateTimer, 1),
4421 IMPORT_SFUNC(KeInitializeDpc, 3),
4422 IMPORT_SFUNC(KeInsertQueueDpc, 3),
4423 IMPORT_SFUNC(KeRemoveQueueDpc, 1),
4424 IMPORT_SFUNC(KeSetImportanceDpc, 2),
4425 IMPORT_SFUNC(KeSetTargetProcessorDpc, 2),
4426 IMPORT_SFUNC(KeFlushQueuedDpcs, 0),
4427 IMPORT_SFUNC(KeGetCurrentProcessorNumber, 1),
4428 IMPORT_SFUNC(ObReferenceObjectByHandle, 6),
4429 IMPORT_FFUNC(ObfDereferenceObject, 1),
4430 IMPORT_SFUNC(ZwClose, 1),
4431 IMPORT_SFUNC(PsCreateSystemThread, 7),
4432 IMPORT_SFUNC(PsTerminateSystemThread, 1),
4433 IMPORT_SFUNC(IoWMIRegistrationControl, 2),
4434 IMPORT_SFUNC(WmiQueryTraceInformation, 5),
4435 IMPORT_CFUNC(WmiTraceMessage, 0),
4436 IMPORT_SFUNC(KeQuerySystemTime, 1),
4437 IMPORT_CFUNC(KeTickCount, 0),
4438 IMPORT_SFUNC(KeDelayExecutionThread, 3),
4439 IMPORT_SFUNC(KeQueryInterruptTime, 0),
4440 IMPORT_SFUNC(KeGetCurrentThread, 0),
4441 IMPORT_SFUNC(KeSetPriorityThread, 2),
4444 * This last entry is a catch-all for any function we haven't
4445 * implemented yet. The PE import list patching routine will
4446 * use it for any function that doesn't have an explicit match
4450 { NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL },
4454 { NULL, NULL, NULL }