1 /* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */
4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
6 * Copyright (c) 2021 The FreeBSD Foundation
8 * Portions of this software were developed by Mark Johnston under sponsorship
9 * from the FreeBSD Foundation.
11 * This code is part of the KMSAN subsystem of the NetBSD kernel.
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 #include <sys/cdefs.h>
39 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $");
42 #include <sys/param.h>
43 #include <sys/systm.h>
48 #include <sys/kernel.h>
49 #include <sys/linker.h>
50 #include <sys/malloc.h>
52 #include <sys/memdesc.h>
55 #include <sys/stack.h>
56 #include <sys/sysctl.h>
62 #include <machine/msan.h>
63 #include <machine/stdarg.h>
65 void kmsan_init_arg(size_t);
66 void kmsan_init_ret(size_t);
68 /* -------------------------------------------------------------------------- */
71 * Part of the compiler ABI.
79 #define MSAN_PARAM_SIZE 800
80 #define MSAN_RETVAL_SIZE 800
82 uint8_t param_shadow[MSAN_PARAM_SIZE];
83 uint8_t retval_shadow[MSAN_RETVAL_SIZE];
84 uint8_t va_arg_shadow[MSAN_PARAM_SIZE];
85 uint8_t va_arg_origin[MSAN_PARAM_SIZE];
86 uint64_t va_arg_overflow_size;
87 msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)];
88 msan_orig_t retval_origin;
91 /* -------------------------------------------------------------------------- */
93 #define MSAN_NCONTEXT 4
94 #define MSAN_ORIG_MASK (~0x3)
96 typedef struct kmsan_td {
98 msan_tls_t tls[MSAN_NCONTEXT];
101 static msan_tls_t dummy_tls;
104 * Use separate dummy regions for loads and stores: stores may mark the region
105 * as uninitialized, and that can trigger false positives.
107 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
108 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
109 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
110 static msan_td_t msan_thread0;
111 static bool kmsan_reporting = false;
114 * Avoid clobbering any thread-local state before we panic.
116 #define kmsan_panic(f, ...) do { \
117 kmsan_disabled = true; \
118 panic(f, __VA_ARGS__); \
121 #define REPORT(f, ...) do { \
122 if (panic_on_violation) { \
123 kmsan_panic(f, __VA_ARGS__); \
128 printf(f "\n", __VA_ARGS__); \
129 stack_print_ddb(&st); \
133 FEATURE(kmsan, "Kernel memory sanitizer");
135 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
138 static bool panic_on_violation = 1;
139 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN,
140 &panic_on_violation, 0,
141 "Panic if an invalid access is detected");
143 static bool kmsan_disabled __read_mostly = true;
144 #define kmsan_enabled (!kmsan_disabled)
145 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
146 &kmsan_disabled, 0, "KMSAN is disabled");
148 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer");
150 /* -------------------------------------------------------------------------- */
152 static inline const char *
153 kmsan_orig_name(int type)
156 case KMSAN_TYPE_STACK:
158 case KMSAN_TYPE_KMEM:
160 case KMSAN_TYPE_MALLOC:
170 kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off,
173 const char *typename;
180 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
183 kmsan_reporting = true;
187 REPORT("MSan: Uninitialized memory in %s, offset %zu",
192 kmsan_md_orig_decode(*orig, &type, &ptr);
193 typename = kmsan_orig_name(type);
195 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
196 sizeof(buf), &foff) == 0) {
197 REPORT("MSan: Uninitialized %s memory in %s, "
198 "offset %zu/%zu, addr %p, from %s+%#lx",
199 typename, hook, off, size, addr, buf, foff);
200 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
202 * The format of the string is: "----var@function". Parse it to
203 * display a nice warning.
205 var = (char *)ptr + 4;
206 strlcpy(buf, var, sizeof(buf));
208 fn = strchr(buf, '@');
210 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, "
211 "variable '%s' from %s", typename, hook, off, var, fn);
213 REPORT("MSan: Uninitialized %s memory in %s, "
214 "offset %zu/%zu, addr %p, PC %p",
215 typename, hook, off, size, addr, (void *)ptr);
220 kmsan_reporting = false;
224 kmsan_report_inline(msan_orig_t orig, unsigned long pc)
226 const char *typename;
233 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
236 kmsan_reporting = true;
240 REPORT("MSan: uninitialized variable in %p", (void *)pc);
244 kmsan_md_orig_decode(orig, &type, &ptr);
245 typename = kmsan_orig_name(type);
247 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
248 sizeof(buf), &foff) == 0) {
249 REPORT("MSan: Uninitialized %s memory from %s+%#lx",
250 typename, buf, foff);
251 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
253 * The format of the string is: "----var@function". Parse it to
254 * display a nice warning.
256 var = (char *)ptr + 4;
257 strlcpy(buf, var, sizeof(buf));
259 fn = strchr(buf, '@');
261 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn);
263 REPORT("MSan: Uninitialized %s memory, origin %x",
269 kmsan_reporting = false;
272 /* -------------------------------------------------------------------------- */
274 static inline msan_meta_t
275 kmsan_meta_get(const void *addr, size_t size, const bool write)
279 if (__predict_false(!kmsan_enabled)) {
280 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
281 ret.orig = (msan_orig_t *)msan_dummy_orig;
282 } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) {
283 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
284 ret.orig = (msan_orig_t *)msan_dummy_orig;
286 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr);
288 (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
289 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig &
297 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size)
302 if (__predict_false(!kmsan_enabled))
304 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr)))
307 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
308 size += ((uintptr_t)orig & (sizeof(*orig) - 1));
309 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
311 for (i = 0; i < size; i += 4) {
317 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size)
321 if (__predict_false(!kmsan_enabled))
323 if (__predict_false(kmsan_md_unsupported(addr)))
326 shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
327 __builtin_memset(shad, c, size);
331 kmsan_meta_copy(void *dst, const void *src, size_t size)
333 uint8_t *orig_src, *orig_dst;
334 uint8_t *shad_src, *shad_dst;
335 msan_orig_t *_src, *_dst;
338 if (__predict_false(!kmsan_enabled))
340 if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst)))
342 if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) {
343 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size);
347 shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src);
348 shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst);
349 __builtin_memmove(shad_dst, shad_src, size);
351 orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src);
352 orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst);
353 for (i = 0; i < size; i++) {
354 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK);
355 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK);
363 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook)
369 if (__predict_false(!kmsan_enabled))
371 if (__predict_false(kmsan_md_unsupported(addr)))
374 shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
375 for (i = 0; i < size; i++) {
376 if (__predict_true(shad[i] == 0))
378 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr + i);
379 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
380 kmsan_report_hook((const char *)addr + i, orig, size, i, hook);
386 kmsan_init_arg(size_t n)
391 if (__predict_false(!kmsan_enabled))
393 if (__predict_false(curthread == NULL))
395 mtd = curthread->td_kmsan;
396 arg = mtd->tls[mtd->ctx].param_shadow;
397 __builtin_memset(arg, 0, n);
401 kmsan_init_ret(size_t n)
406 if (__predict_false(!kmsan_enabled))
408 if (__predict_false(curthread == NULL))
410 mtd = curthread->td_kmsan;
411 arg = mtd->tls[mtd->ctx].retval_shadow;
412 __builtin_memset(arg, 0, n);
416 kmsan_check_arg(size_t size, const char *hook)
423 if (__predict_false(!kmsan_enabled))
425 if (__predict_false(curthread == NULL))
427 mtd = curthread->td_kmsan;
429 arg = mtd->tls[ctx].param_shadow;
431 for (i = 0; i < size; i++) {
432 if (__predict_true(arg[i] == 0))
434 orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)];
435 kmsan_report_hook((const char *)arg + i, orig, size, i, hook);
441 kmsan_thread_alloc(struct thread *td)
450 /* We might be recycling a thread. */
451 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) +
453 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK);
455 __builtin_memset(mtd, 0, sizeof(*mtd));
458 if (td->td_kstack != 0)
459 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages),
466 kmsan_thread_free(struct thread *td)
472 if (__predict_false(td == curthread))
473 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__);
476 kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *));
481 void kmsan_intr_enter(void);
482 void kmsan_intr_leave(void);
485 kmsan_intr_enter(void)
489 if (__predict_false(!kmsan_enabled))
492 mtd = curthread->td_kmsan;
494 if (__predict_false(mtd->ctx >= MSAN_NCONTEXT))
495 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
499 kmsan_intr_leave(void)
503 if (__predict_false(!kmsan_enabled))
506 mtd = curthread->td_kmsan;
507 if (__predict_false(mtd->ctx == 0))
508 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
512 /* -------------------------------------------------------------------------- */
515 kmsan_shadow_map(vm_offset_t addr, size_t size)
520 MPASS(addr % PAGE_SIZE == 0);
521 MPASS(size % PAGE_SIZE == 0);
528 va = kmsan_md_addr_to_shad(addr);
529 for (i = 0; i < npages; i++) {
530 pmap_san_enter(va + ptoa(i));
533 va = kmsan_md_addr_to_orig(addr);
534 for (i = 0; i < npages; i++) {
535 pmap_san_enter(va + ptoa(i));
540 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc)
544 orig = kmsan_md_orig_encode(type, pc);
545 kmsan_origin_fill(addr, orig, size);
549 kmsan_mark(const void *addr, size_t size, uint8_t c)
551 kmsan_shadow_fill((uintptr_t)addr, c, size);
555 kmsan_mark_bio(const struct bio *bp, uint8_t c)
557 kmsan_mark(bp->bio_data, bp->bio_length, c);
561 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c)
564 if ((m->m_flags & M_EXTPG) == 0)
565 kmsan_mark(m->m_data, m->m_len, c);
571 kmsan_check(const void *p, size_t sz, const char *descr)
573 kmsan_shadow_check((uintptr_t)p, sz, descr);
577 kmsan_check_bio(const struct bio *bp, const char *descr)
579 kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr);
583 kmsan_check_mbuf(const struct mbuf *m, const char *descr)
586 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr);
587 } while ((m = m->m_next) != NULL);
596 TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled);
600 /* Initialize the TLS for curthread. */
601 msan_thread0.ctx = 0;
602 thread0.td_kmsan = &msan_thread0;
604 /* Now officially enabled. */
605 kmsan_disabled = false;
608 /* -------------------------------------------------------------------------- */
610 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
611 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
614 __msan_metadata_ptr_for_load_n(void *addr, size_t size)
616 return (kmsan_meta_get(addr, size, false));
620 __msan_metadata_ptr_for_store_n(void *addr, size_t size)
622 return (kmsan_meta_get(addr, size, true));
625 #define MSAN_META_FUNC(size) \
626 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \
627 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \
629 return (kmsan_meta_get(addr, size, false)); \
631 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \
632 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \
634 return (kmsan_meta_get(addr, size, true)); \
642 void __msan_instrument_asm_store(const void *, size_t);
643 msan_orig_t __msan_chain_origin(msan_orig_t);
644 void __msan_poison(const void *, size_t);
645 void __msan_unpoison(const void *, size_t);
646 void __msan_poison_alloca(const void *, uint64_t, const char *);
647 void __msan_unpoison_alloca(const void *, uint64_t);
648 void __msan_warning(msan_orig_t);
649 msan_tls_t *__msan_get_context_state(void);
652 __msan_instrument_asm_store(const void *addr, size_t size)
654 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
658 __msan_chain_origin(msan_orig_t origin)
664 __msan_poison(const void *addr, size_t size)
666 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
670 __msan_unpoison(const void *addr, size_t size)
672 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
676 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr)
680 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
681 kmsan_origin_fill(addr, orig, size);
682 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
686 __msan_unpoison_alloca(const void *addr, uint64_t size)
688 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
692 __msan_warning(msan_orig_t origin)
694 if (__predict_false(!kmsan_enabled))
696 kmsan_report_inline(origin, KMSAN_RET_ADDR);
700 __msan_get_context_state(void)
705 * When APs are started, they execute some C code before curthread is
706 * set. We have to handle that here.
708 if (__predict_false(!kmsan_enabled || curthread == NULL))
710 mtd = curthread->td_kmsan;
711 return (&mtd->tls[mtd->ctx]);
714 /* -------------------------------------------------------------------------- */
717 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
718 * initialized areas properly.
722 kmsan_memcpy(void *dst, const void *src, size_t len)
724 /* No kmsan_check_arg, because inlined. */
725 kmsan_init_ret(sizeof(void *));
726 if (__predict_true(len != 0)) {
727 kmsan_meta_copy(dst, src, len);
729 return (__builtin_memcpy(dst, src, len));
733 kmsan_memcmp(const void *b1, const void *b2, size_t len)
735 const uint8_t *_b1 = b1, *_b2 = b2;
738 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len),
740 kmsan_init_ret(sizeof(int));
742 for (i = 0; i < len; i++) {
744 kmsan_shadow_check((uintptr_t)b1, i + 1,
746 kmsan_shadow_check((uintptr_t)b2, i + 1,
748 return (*_b1 - *_b2);
757 kmsan_memset(void *dst, int c, size_t len)
759 /* No kmsan_check_arg, because inlined. */
760 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len);
761 kmsan_init_ret(sizeof(void *));
762 return (__builtin_memset(dst, c, len));
766 kmsan_memmove(void *dst, const void *src, size_t len)
768 /* No kmsan_check_arg, because inlined. */
769 if (__predict_true(len != 0)) {
770 kmsan_meta_copy(dst, src, len);
772 kmsan_init_ret(sizeof(void *));
773 return (__builtin_memmove(dst, src, len));
776 __strong_reference(kmsan_memcpy, __msan_memcpy);
777 __strong_reference(kmsan_memset, __msan_memset);
778 __strong_reference(kmsan_memmove, __msan_memmove);
781 kmsan_strcpy(char *dst, const char *src)
783 const char *_src = src;
787 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args");
797 kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2");
798 kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len);
799 kmsan_init_ret(sizeof(char *));
804 kmsan_strcmp(const char *s1, const char *s2)
806 const char *_s1 = s1, *_s2 = s2;
809 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args");
810 kmsan_init_ret(sizeof(int));
817 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
818 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
824 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
825 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
827 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
831 kmsan_strlen(const char *str)
835 kmsan_check_arg(sizeof(str), "strlen():args");
844 kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1");
845 kmsan_init_ret(sizeof(size_t));
849 int kmsan_copyin(const void *, void *, size_t);
850 int kmsan_copyout(const void *, void *, size_t);
851 int kmsan_copyinstr(const void *, void *, size_t, size_t *);
854 kmsan_copyin(const void *uaddr, void *kaddr, size_t len)
858 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len),
860 ret = copyin(uaddr, kaddr, len);
862 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len);
863 kmsan_init_ret(sizeof(int));
868 kmsan_copyout(const void *kaddr, void *uaddr, size_t len)
870 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len),
872 kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1");
873 kmsan_init_ret(sizeof(int));
874 return (copyout(kaddr, uaddr, len));
878 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
883 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) +
884 sizeof(len) + sizeof(done), "copyinstr():args");
885 ret = copyinstr(uaddr, kaddr, len, &_done);
887 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done);
890 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t));
892 kmsan_init_ret(sizeof(int));
896 /* -------------------------------------------------------------------------- */
899 kmsan_fubyte(volatile const void *base)
903 kmsan_check_arg(sizeof(base), "fubyte(): args");
905 kmsan_init_ret(sizeof(int));
910 kmsan_fuword16(volatile const void *base)
914 kmsan_check_arg(sizeof(base), "fuword16(): args");
915 ret = fuword16(base);
916 kmsan_init_ret(sizeof(int));
921 kmsan_fueword(volatile const void *base, long *val)
925 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args");
926 ret = fueword(base, val);
928 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
930 kmsan_init_ret(sizeof(int));
935 kmsan_fueword32(volatile const void *base, int32_t *val)
939 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args");
940 ret = fueword32(base, val);
942 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
944 kmsan_init_ret(sizeof(int));
949 kmsan_fueword64(volatile const void *base, int64_t *val)
953 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args");
954 ret = fueword64(base, val);
956 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
958 kmsan_init_ret(sizeof(int));
963 kmsan_subyte(volatile void *base, int byte)
967 kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args");
968 ret = subyte(base, byte);
969 kmsan_init_ret(sizeof(int));
974 kmsan_suword(volatile void *base, long word)
978 kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args");
979 ret = suword(base, word);
980 kmsan_init_ret(sizeof(int));
985 kmsan_suword16(volatile void *base, int word)
989 kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args");
990 ret = suword16(base, word);
991 kmsan_init_ret(sizeof(int));
996 kmsan_suword32(volatile void *base, int32_t word)
1000 kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args");
1001 ret = suword32(base, word);
1002 kmsan_init_ret(sizeof(int));
1007 kmsan_suword64(volatile void *base, int64_t word)
1011 kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args");
1012 ret = suword64(base, word);
1013 kmsan_init_ret(sizeof(int));
1018 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
1023 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1024 sizeof(newval), "casueword32(): args");
1025 ret = casueword32(base, oldval, oldvalp, newval);
1026 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1028 kmsan_init_ret(sizeof(int));
1033 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
1038 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1039 sizeof(newval), "casueword32(): args");
1040 ret = casueword(base, oldval, oldvalp, newval);
1041 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1043 kmsan_init_ret(sizeof(int));
1047 /* -------------------------------------------------------------------------- */
1049 #include <machine/atomic.h>
1050 #include <sys/atomic_san.h>
1052 #define _MSAN_ATOMIC_FUNC_ADD(name, type) \
1053 void kmsan_atomic_add_##name(volatile type *ptr, type val) \
1055 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1056 "atomic_add_" #name "():args"); \
1057 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1058 "atomic_add_" #name "():ptr"); \
1059 atomic_add_##name(ptr, val); \
1062 #define MSAN_ATOMIC_FUNC_ADD(name, type) \
1063 _MSAN_ATOMIC_FUNC_ADD(name, type) \
1064 _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \
1065 _MSAN_ATOMIC_FUNC_ADD(rel_##name, type)
1067 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1068 void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \
1070 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1071 "atomic_subtract_" #name "():args"); \
1072 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1073 "atomic_subtract_" #name "():ptr"); \
1074 atomic_subtract_##name(ptr, val); \
1077 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1078 _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1079 _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
1080 _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
1082 #define _MSAN_ATOMIC_FUNC_SET(name, type) \
1083 void kmsan_atomic_set_##name(volatile type *ptr, type val) \
1085 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1086 "atomic_set_" #name "():args"); \
1087 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1088 "atomic_set_" #name "():ptr"); \
1089 atomic_set_##name(ptr, val); \
1092 #define MSAN_ATOMIC_FUNC_SET(name, type) \
1093 _MSAN_ATOMIC_FUNC_SET(name, type) \
1094 _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \
1095 _MSAN_ATOMIC_FUNC_SET(rel_##name, type)
1097 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1098 void kmsan_atomic_clear_##name(volatile type *ptr, type val) \
1100 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1101 "atomic_clear_" #name "():args"); \
1102 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1103 "atomic_clear_" #name "():ptr"); \
1104 atomic_clear_##name(ptr, val); \
1107 #define MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1108 _MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1109 _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
1110 _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
1112 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \
1113 type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
1115 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1116 "atomic_fetchadd_" #name "():args"); \
1117 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1118 "atomic_fetchadd_" #name "():ptr"); \
1119 kmsan_init_ret(sizeof(type)); \
1120 return (atomic_fetchadd_##name(ptr, val)); \
1123 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
1124 type kmsan_atomic_readandclear_##name(volatile type *ptr) \
1126 kmsan_check_arg(sizeof(ptr), \
1127 "atomic_readandclear_" #name "():args"); \
1128 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1129 "atomic_readandclear_" #name "():ptr"); \
1130 kmsan_init_ret(sizeof(type)); \
1131 return (atomic_readandclear_##name(ptr)); \
1134 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
1135 int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
1137 kmsan_check_arg(sizeof(ptr) + sizeof(v), \
1138 "atomic_testandclear_" #name "():args"); \
1139 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1140 "atomic_testandclear_" #name "():ptr"); \
1141 kmsan_init_ret(sizeof(int)); \
1142 return (atomic_testandclear_##name(ptr, v)); \
1145 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \
1146 int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \
1148 kmsan_check_arg(sizeof(ptr) + sizeof(v), \
1149 "atomic_testandset_" #name "():args"); \
1150 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1151 "atomic_testandset_" #name "():ptr"); \
1152 kmsan_init_ret(sizeof(int)); \
1153 return (atomic_testandset_##name(ptr, v)); \
1156 #define MSAN_ATOMIC_FUNC_SWAP(name, type) \
1157 type kmsan_atomic_swap_##name(volatile type *ptr, type val) \
1159 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1160 "atomic_swap_" #name "():args"); \
1161 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1162 "atomic_swap_" #name "():ptr"); \
1163 kmsan_init_ret(sizeof(type)); \
1164 return (atomic_swap_##name(ptr, val)); \
1167 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1168 int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \
1171 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \
1172 sizeof(nval), "atomic_cmpset_" #name "():args"); \
1173 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1174 "atomic_cmpset_" #name "():ptr"); \
1175 kmsan_init_ret(sizeof(int)); \
1176 return (atomic_cmpset_##name(ptr, oval, nval)); \
1179 #define MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1180 _MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1181 _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
1182 _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
1184 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1185 int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
1188 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \
1189 sizeof(nval), "atomic_fcmpset_" #name "():args"); \
1190 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1191 "atomic_fcmpset_" #name "():ptr"); \
1192 kmsan_init_ret(sizeof(int)); \
1193 return (atomic_fcmpset_##name(ptr, oval, nval)); \
1196 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1197 _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1198 _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
1199 _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
1201 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \
1202 void kmsan_atomic_thread_fence_##name(void) \
1204 atomic_thread_fence_##name(); \
1207 #define _MSAN_ATOMIC_FUNC_LOAD(name, type) \
1208 type kmsan_atomic_load_##name(volatile type *ptr) \
1210 kmsan_check_arg(sizeof(ptr), \
1211 "atomic_load_" #name "():args"); \
1212 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1213 "atomic_load_" #name "():ptr"); \
1214 kmsan_init_ret(sizeof(type)); \
1215 return (atomic_load_##name(ptr)); \
1218 #define MSAN_ATOMIC_FUNC_LOAD(name, type) \
1219 _MSAN_ATOMIC_FUNC_LOAD(name, type) \
1220 _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type)
1222 #define _MSAN_ATOMIC_FUNC_STORE(name, type) \
1223 void kmsan_atomic_store_##name(volatile type *ptr, type val) \
1225 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1226 "atomic_store_" #name "():args"); \
1227 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \
1229 atomic_store_##name(ptr, val); \
1232 #define MSAN_ATOMIC_FUNC_STORE(name, type) \
1233 _MSAN_ATOMIC_FUNC_STORE(name, type) \
1234 _MSAN_ATOMIC_FUNC_STORE(rel_##name, type)
1236 MSAN_ATOMIC_FUNC_ADD(8, uint8_t);
1237 MSAN_ATOMIC_FUNC_ADD(16, uint16_t);
1238 MSAN_ATOMIC_FUNC_ADD(32, uint32_t);
1239 MSAN_ATOMIC_FUNC_ADD(64, uint64_t);
1240 MSAN_ATOMIC_FUNC_ADD(int, u_int);
1241 MSAN_ATOMIC_FUNC_ADD(long, u_long);
1242 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
1244 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
1245 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
1246 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
1247 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
1248 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
1249 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
1250 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
1252 MSAN_ATOMIC_FUNC_SET(8, uint8_t);
1253 MSAN_ATOMIC_FUNC_SET(16, uint16_t);
1254 MSAN_ATOMIC_FUNC_SET(32, uint32_t);
1255 MSAN_ATOMIC_FUNC_SET(64, uint64_t);
1256 MSAN_ATOMIC_FUNC_SET(int, u_int);
1257 MSAN_ATOMIC_FUNC_SET(long, u_long);
1258 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
1260 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
1261 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
1262 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
1263 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
1264 MSAN_ATOMIC_FUNC_CLEAR(int, u_int);
1265 MSAN_ATOMIC_FUNC_CLEAR(long, u_long);
1266 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
1268 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
1269 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
1270 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int);
1271 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long);
1273 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
1274 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
1275 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
1276 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
1277 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
1279 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
1280 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
1281 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
1282 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
1284 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
1285 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
1286 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
1287 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
1289 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t);
1290 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t);
1291 MSAN_ATOMIC_FUNC_SWAP(int, u_int);
1292 MSAN_ATOMIC_FUNC_SWAP(long, u_long);
1293 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
1295 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
1296 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
1297 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
1298 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
1299 MSAN_ATOMIC_FUNC_CMPSET(int, u_int);
1300 MSAN_ATOMIC_FUNC_CMPSET(long, u_long);
1301 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
1303 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
1304 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
1305 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
1306 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
1307 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int);
1308 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long);
1309 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
1311 _MSAN_ATOMIC_FUNC_LOAD(bool, bool);
1312 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t);
1313 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t);
1314 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t);
1315 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t);
1316 MSAN_ATOMIC_FUNC_LOAD(char, u_char);
1317 MSAN_ATOMIC_FUNC_LOAD(short, u_short);
1318 MSAN_ATOMIC_FUNC_LOAD(int, u_int);
1319 MSAN_ATOMIC_FUNC_LOAD(long, u_long);
1320 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
1322 _MSAN_ATOMIC_FUNC_STORE(bool, bool);
1323 MSAN_ATOMIC_FUNC_STORE(8, uint8_t);
1324 MSAN_ATOMIC_FUNC_STORE(16, uint16_t);
1325 MSAN_ATOMIC_FUNC_STORE(32, uint32_t);
1326 MSAN_ATOMIC_FUNC_STORE(64, uint64_t);
1327 MSAN_ATOMIC_FUNC_STORE(char, u_char);
1328 MSAN_ATOMIC_FUNC_STORE(short, u_short);
1329 MSAN_ATOMIC_FUNC_STORE(int, u_int);
1330 MSAN_ATOMIC_FUNC_STORE(long, u_long);
1331 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
1333 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq);
1334 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel);
1335 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
1336 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
1339 kmsan_atomic_interrupt_fence(void)
1341 atomic_interrupt_fence();
1344 /* -------------------------------------------------------------------------- */
1346 #include <sys/bus.h>
1347 #include <machine/bus.h>
1348 #include <sys/bus_san.h>
1351 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
1352 int flags, bus_space_handle_t *handlep)
1354 return (bus_space_map(tag, hnd, size, flags, handlep));
1358 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
1361 bus_space_unmap(tag, hnd, size);
1365 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
1366 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
1368 return (bus_space_subregion(tag, hnd, offset, size, handlep));
1372 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
1375 bus_space_free(tag, hnd, size);
1379 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
1380 bus_size_t offset, bus_size_t size, int flags)
1382 bus_space_barrier(tag, hnd, offset, size, flags);
1385 /* XXXMJ x86-specific */
1386 #define MSAN_BUS_READ_FUNC(func, width, type) \
1387 type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \
1388 bus_space_handle_t hnd, bus_size_t offset) \
1391 if ((tag) != X86_BUS_SPACE_IO) \
1392 kmsan_shadow_fill((uintptr_t)(hnd + offset), \
1393 KMSAN_STATE_INITED, (width)); \
1394 ret = bus_space_read##func##_##width(tag, hnd, offset); \
1395 kmsan_init_ret(sizeof(type)); \
1399 #define MSAN_BUS_READ_PTR_FUNC(func, width, type) \
1400 void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
1401 bus_space_handle_t hnd, bus_size_t size, type *buf, \
1404 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \
1406 bus_space_read_##func##_##width(tag, hnd, size, buf, \
1410 MSAN_BUS_READ_FUNC(, 1, uint8_t)
1411 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
1412 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
1413 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
1414 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
1415 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
1417 MSAN_BUS_READ_FUNC(, 2, uint16_t)
1418 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
1419 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
1420 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
1421 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
1422 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
1424 MSAN_BUS_READ_FUNC(, 4, uint32_t)
1425 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
1426 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
1427 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
1428 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
1429 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
1431 MSAN_BUS_READ_FUNC(, 8, uint64_t)
1433 #define MSAN_BUS_WRITE_FUNC(func, width, type) \
1434 void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
1435 bus_space_handle_t hnd, bus_size_t offset, type value) \
1437 bus_space_write##func##_##width(tag, hnd, offset, value);\
1440 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \
1441 void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
1442 bus_space_handle_t hnd, bus_size_t size, const type *buf, \
1445 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
1446 "bus_space_write()"); \
1447 bus_space_write_##func##_##width(tag, hnd, size, buf, \
1451 MSAN_BUS_WRITE_FUNC(, 1, uint8_t)
1452 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
1453 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
1454 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
1455 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
1456 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
1458 MSAN_BUS_WRITE_FUNC(, 2, uint16_t)
1459 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
1460 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
1461 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
1462 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1463 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1465 MSAN_BUS_WRITE_FUNC(, 4, uint32_t)
1466 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1467 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1468 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1469 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1470 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1472 MSAN_BUS_WRITE_FUNC(, 8, uint64_t)
1474 #define MSAN_BUS_SET_FUNC(func, width, type) \
1475 void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
1476 bus_space_handle_t hnd, bus_size_t offset, type value, \
1479 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1483 MSAN_BUS_SET_FUNC(multi, 1, uint8_t)
1484 MSAN_BUS_SET_FUNC(region, 1, uint8_t)
1485 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1486 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1488 MSAN_BUS_SET_FUNC(multi, 2, uint16_t)
1489 MSAN_BUS_SET_FUNC(region, 2, uint16_t)
1490 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1491 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1493 MSAN_BUS_SET_FUNC(multi, 4, uint32_t)
1494 MSAN_BUS_SET_FUNC(region, 4, uint32_t)
1495 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1496 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1498 /* -------------------------------------------------------------------------- */
1501 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op)
1504 * Some drivers, e.g., nvme, use the same code path for loading device
1505 * read and write requests, and will thus specify both flags. In this
1506 * case we should not do any checking since it will generally lead to
1509 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) ==
1510 BUS_DMASYNC_PREWRITE) {
1511 switch (desc->md_type) {
1513 kmsan_check(desc->u.md_vaddr, desc->md_len,
1517 kmsan_check_mbuf(desc->u.md_mbuf, "dmasync");
1522 kmsan_panic("%s: unhandled memdesc type %d", __func__,
1526 if ((op & BUS_DMASYNC_POSTREAD) != 0) {
1527 switch (desc->md_type) {
1529 kmsan_mark(desc->u.md_vaddr, desc->md_len,
1530 KMSAN_STATE_INITED);
1533 kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED);
1538 kmsan_panic("%s: unhandled memdesc type %d", __func__,