1 /* $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $ */
4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
7 * This code is part of the KASAN subsystem of the NetBSD kernel.
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $");
38 #include <sys/param.h>
39 #include <sys/systm.h>
41 #include <sys/kernel.h>
42 #include <sys/stack.h>
43 #include <sys/sysctl.h>
45 #include <machine/asan.h>
46 #include <machine/bus.h>
48 /* ASAN constants. Part of the compiler ABI. */
49 #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE - 1)
50 #define KASAN_ALLOCA_SCALE_SIZE 32
52 /* ASAN ABI version. */
53 #if defined(__clang__) && (__clang_major__ - 0 >= 6)
54 #define ASAN_ABI_VERSION 8
55 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
56 #define ASAN_ABI_VERSION 8
57 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
58 #define ASAN_ABI_VERSION 6
60 #error "Unsupported compiler version"
63 #define __RET_ADDR (unsigned long)__builtin_return_address(0)
65 /* Global variable descriptor. Part of the compiler ABI. */
66 struct __asan_global_source_location {
72 struct __asan_global {
73 const void *beg; /* address of the global variable */
74 size_t size; /* size of the global variable */
75 size_t size_with_redzone; /* size with the redzone */
76 const void *name; /* name of the variable */
77 const void *module_name; /* name of the module where the var is declared */
78 unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
79 struct __asan_global_source_location *location;
80 #if ASAN_ABI_VERSION >= 7
81 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */
85 FEATURE(kasan, "Kernel address sanitizer");
87 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
90 static int panic_on_violation = 1;
91 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN,
92 &panic_on_violation, 0,
93 "Panic if an invalid access is detected");
95 #define kasan_enabled (!kasan_disabled)
96 static bool kasan_disabled __read_mostly = true;
97 SYSCTL_BOOL(_debug_kasan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
98 &kasan_disabled, 0, "KASAN is disabled");
100 /* -------------------------------------------------------------------------- */
103 kasan_shadow_map(vm_offset_t addr, size_t size)
105 size_t sz, npages, i;
106 vm_offset_t sva, eva;
108 KASSERT(addr % KASAN_SHADOW_SCALE == 0,
109 ("%s: invalid address %#lx", __func__, addr));
111 sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE;
113 sva = kasan_md_addr_to_shad(addr);
114 eva = kasan_md_addr_to_shad(addr) + sz;
116 sva = rounddown(sva, PAGE_SIZE);
117 eva = roundup(eva, PAGE_SIZE);
119 npages = (eva - sva) / PAGE_SIZE;
121 KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS,
122 ("%s: invalid address range %#lx-%#lx", __func__, sva, eva));
124 for (i = 0; i < npages; i++)
125 pmap_san_enter(sva + ptoa(i));
134 TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled);
138 /* MD initialization. */
141 /* Now officially enabled. */
142 kasan_disabled = false;
146 kasan_init_early(vm_offset_t stack, size_t size)
148 kasan_md_init_early(stack, size);
151 static inline const char *
152 kasan_code_name(uint8_t code)
155 case KASAN_GENERIC_REDZONE:
156 return "GenericRedZone";
157 case KASAN_MALLOC_REDZONE:
158 return "MallocRedZone";
159 case KASAN_KMEM_REDZONE:
160 return "KmemRedZone";
161 case KASAN_UMA_FREED:
162 return "UMAUseAfterFree";
163 case KASAN_KSTACK_FREED:
164 return "KernelStack";
165 case KASAN_EXEC_ARGS_FREED:
168 return "RedZonePartial";
169 case KASAN_STACK_LEFT:
171 case KASAN_STACK_MID:
172 return "StackMiddle";
173 case KASAN_STACK_RIGHT:
175 case KASAN_USE_AFTER_RET:
176 return "UseAfterRet";
177 case KASAN_USE_AFTER_SCOPE:
178 return "UseAfterScope";
184 #define REPORT(f, ...) do { \
185 if (panic_on_violation) { \
186 kasan_disabled = true; \
187 panic(f, __VA_ARGS__); \
192 printf(f "\n", __VA_ARGS__); \
193 stack_print_ddb(&st); \
198 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
201 REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)",
202 size, (write ? "write" : "read"), addr, kasan_code_name(code),
206 static __always_inline void
207 kasan_shadow_1byte_markvalid(unsigned long addr)
209 int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
210 int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
215 static __always_inline void
216 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
220 for (i = 0; i < size; i++) {
221 kasan_shadow_1byte_markvalid((unsigned long)addr + i);
225 static __always_inline void
226 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
230 if (__predict_false(size == 0))
232 if (__predict_false(kasan_md_unsupported((vm_offset_t)addr)))
235 KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
236 ("%s: invalid address %p", __func__, addr));
237 KASSERT(size % KASAN_SHADOW_SCALE == 0,
238 ("%s: invalid size %zu", __func__, size));
240 shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr);
241 size = size >> KASAN_SHADOW_SCALE_SHIFT;
243 __builtin_memset(shad, code, size);
247 * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
248 * and the rest as invalid. There are generally two use cases:
250 * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
251 * the redzone at the end of the buffer as invalid. If the entire is to be
252 * marked invalid, origsize will be 0.
254 * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
257 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
262 if (__predict_false(!kasan_enabled))
265 if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS &&
266 (vm_offset_t)addr < DMAP_MAX_ADDRESS)
269 KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS &&
270 (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS,
271 ("%s: invalid address %p", __func__, addr));
272 KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
273 ("%s: invalid address %p", __func__, addr));
274 redz = redzsize - roundup(size, KASAN_SHADOW_SCALE);
275 KASSERT(redz % KASAN_SHADOW_SCALE == 0,
276 ("%s: invalid size %zu", __func__, redz));
277 shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr);
279 /* Chunks of 8 bytes, valid. */
280 n = size / KASAN_SHADOW_SCALE;
281 for (i = 0; i < n; i++) {
285 /* Possibly one chunk, mid. */
286 if ((size & KASAN_SHADOW_MASK) != 0) {
287 *shad++ = (size & KASAN_SHADOW_MASK);
290 /* Chunks of 8 bytes, invalid. */
291 n = redz / KASAN_SHADOW_SCALE;
292 for (i = 0; i < n; i++) {
297 /* -------------------------------------------------------------------------- */
299 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \
300 (addr >> KASAN_SHADOW_SCALE_SHIFT) != \
301 ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
303 static __always_inline bool
304 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
306 int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
307 int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
309 if (__predict_true(*byte == 0 || last <= *byte)) {
316 static __always_inline bool
317 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
321 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
322 return (kasan_shadow_1byte_isvalid(addr, code) &&
323 kasan_shadow_1byte_isvalid(addr+1, code));
326 byte = (int8_t *)kasan_md_addr_to_shad(addr);
327 last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
329 if (__predict_true(*byte == 0 || last <= *byte)) {
336 static __always_inline bool
337 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
341 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
342 return (kasan_shadow_2byte_isvalid(addr, code) &&
343 kasan_shadow_2byte_isvalid(addr+2, code));
346 byte = (int8_t *)kasan_md_addr_to_shad(addr);
347 last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
349 if (__predict_true(*byte == 0 || last <= *byte)) {
356 static __always_inline bool
357 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
361 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
362 return (kasan_shadow_4byte_isvalid(addr, code) &&
363 kasan_shadow_4byte_isvalid(addr+4, code));
366 byte = (int8_t *)kasan_md_addr_to_shad(addr);
367 last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
369 if (__predict_true(*byte == 0 || last <= *byte)) {
376 static __always_inline bool
377 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
381 for (i = 0; i < size; i++) {
382 if (!kasan_shadow_1byte_isvalid(addr+i, code))
389 static __always_inline void
390 kasan_shadow_check(unsigned long addr, size_t size, bool write,
391 unsigned long retaddr)
396 if (__predict_false(!kasan_enabled))
398 if (__predict_false(size == 0))
400 if (__predict_false(kasan_md_unsupported(addr)))
402 if (KERNEL_PANICKED())
405 if (__builtin_constant_p(size)) {
408 valid = kasan_shadow_1byte_isvalid(addr, &code);
411 valid = kasan_shadow_2byte_isvalid(addr, &code);
414 valid = kasan_shadow_4byte_isvalid(addr, &code);
417 valid = kasan_shadow_8byte_isvalid(addr, &code);
420 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
424 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
427 if (__predict_false(!valid)) {
428 kasan_report(addr, size, write, retaddr, code);
432 /* -------------------------------------------------------------------------- */
435 kasan_memcpy(void *dst, const void *src, size_t len)
437 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
438 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
439 return (__builtin_memcpy(dst, src, len));
443 kasan_memcmp(const void *b1, const void *b2, size_t len)
445 kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
446 kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
447 return (__builtin_memcmp(b1, b2, len));
451 kasan_memset(void *b, int c, size_t len)
453 kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
454 return (__builtin_memset(b, c, len));
458 kasan_memmove(void *dst, const void *src, size_t len)
460 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
461 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
462 return (__builtin_memmove(dst, src, len));
466 kasan_strlen(const char *str)
472 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
482 kasan_strcpy(char *dst, const char *src)
487 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
488 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
499 kasan_strcmp(const char *s1, const char *s2)
502 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
503 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
511 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
515 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
517 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
518 return (copyin(uaddr, kaddr, len));
522 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
524 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
525 return (copyinstr(uaddr, kaddr, len, done));
529 kasan_copyout(const void *kaddr, void *uaddr, size_t len)
531 kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
532 return (copyout(kaddr, uaddr, len));
535 /* -------------------------------------------------------------------------- */
538 kasan_fubyte(volatile const void *base)
540 return (fubyte(base));
544 kasan_fuword16(volatile const void *base)
546 return (fuword16(base));
550 kasan_fueword(volatile const void *base, long *val)
552 kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
553 return (fueword(base, val));
557 kasan_fueword32(volatile const void *base, int32_t *val)
559 kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
560 return (fueword32(base, val));
564 kasan_fueword64(volatile const void *base, int64_t *val)
566 kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
567 return (fueword64(base, val));
571 kasan_subyte(volatile void *base, int byte)
573 return (subyte(base, byte));
577 kasan_suword(volatile void *base, long word)
579 return (suword(base, word));
583 kasan_suword16(volatile void *base, int word)
585 return (suword16(base, word));
589 kasan_suword32(volatile void *base, int32_t word)
591 return (suword32(base, word));
595 kasan_suword64(volatile void *base, int64_t word)
597 return (suword64(base, word));
601 kasan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
604 kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
606 return (casueword32(base, oldval, oldvalp, newval));
610 kasan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
613 kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
615 return (casueword(base, oldval, oldvalp, newval));
618 /* -------------------------------------------------------------------------- */
620 #include <machine/atomic.h>
621 #include <sys/atomic_san.h>
623 #define _ASAN_ATOMIC_FUNC_ADD(name, type) \
624 void kasan_atomic_add_##name(volatile type *ptr, type val) \
626 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
628 atomic_add_##name(ptr, val); \
631 #define ASAN_ATOMIC_FUNC_ADD(name, type) \
632 _ASAN_ATOMIC_FUNC_ADD(name, type) \
633 _ASAN_ATOMIC_FUNC_ADD(acq_##name, type) \
634 _ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
636 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
637 void kasan_atomic_subtract_##name(volatile type *ptr, type val) \
639 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
641 atomic_subtract_##name(ptr, val); \
644 #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
645 _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
646 _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
647 _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
649 #define _ASAN_ATOMIC_FUNC_SET(name, type) \
650 void kasan_atomic_set_##name(volatile type *ptr, type val) \
652 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
654 atomic_set_##name(ptr, val); \
657 #define ASAN_ATOMIC_FUNC_SET(name, type) \
658 _ASAN_ATOMIC_FUNC_SET(name, type) \
659 _ASAN_ATOMIC_FUNC_SET(acq_##name, type) \
660 _ASAN_ATOMIC_FUNC_SET(rel_##name, type)
662 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type) \
663 void kasan_atomic_clear_##name(volatile type *ptr, type val) \
665 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
667 atomic_clear_##name(ptr, val); \
670 #define ASAN_ATOMIC_FUNC_CLEAR(name, type) \
671 _ASAN_ATOMIC_FUNC_CLEAR(name, type) \
672 _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
673 _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
675 #define ASAN_ATOMIC_FUNC_FETCHADD(name, type) \
676 type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \
678 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
680 return (atomic_fetchadd_##name(ptr, val)); \
683 #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
684 type kasan_atomic_readandclear_##name(volatile type *ptr) \
686 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
688 return (atomic_readandclear_##name(ptr)); \
691 #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
692 int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
694 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
696 return (atomic_testandclear_##name(ptr, v)); \
699 #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type) \
700 int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
702 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
704 return (atomic_testandset_##name(ptr, v)); \
707 #define ASAN_ATOMIC_FUNC_SWAP(name, type) \
708 type kasan_atomic_swap_##name(volatile type *ptr, type val) \
710 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
712 return (atomic_swap_##name(ptr, val)); \
715 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type) \
716 int kasan_atomic_cmpset_##name(volatile type *ptr, type oval, \
719 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
721 return (atomic_cmpset_##name(ptr, oval, nval)); \
724 #define ASAN_ATOMIC_FUNC_CMPSET(name, type) \
725 _ASAN_ATOMIC_FUNC_CMPSET(name, type) \
726 _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
727 _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
729 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
730 int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
733 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
735 return (atomic_fcmpset_##name(ptr, oval, nval)); \
738 #define ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
739 _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
740 _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
741 _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
743 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name) \
744 void kasan_atomic_thread_fence_##name(void) \
746 atomic_thread_fence_##name(); \
749 #define _ASAN_ATOMIC_FUNC_LOAD(name, type) \
750 type kasan_atomic_load_##name(volatile type *ptr) \
752 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
754 return (atomic_load_##name(ptr)); \
757 #define ASAN_ATOMIC_FUNC_LOAD(name, type) \
758 _ASAN_ATOMIC_FUNC_LOAD(name, type) \
759 _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
761 #define _ASAN_ATOMIC_FUNC_STORE(name, type) \
762 void kasan_atomic_store_##name(volatile type *ptr, type val) \
764 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
766 atomic_store_##name(ptr, val); \
769 #define ASAN_ATOMIC_FUNC_STORE(name, type) \
770 _ASAN_ATOMIC_FUNC_STORE(name, type) \
771 _ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
773 ASAN_ATOMIC_FUNC_ADD(8, uint8_t);
774 ASAN_ATOMIC_FUNC_ADD(16, uint16_t);
775 ASAN_ATOMIC_FUNC_ADD(32, uint32_t);
776 ASAN_ATOMIC_FUNC_ADD(64, uint64_t);
777 ASAN_ATOMIC_FUNC_ADD(int, u_int);
778 ASAN_ATOMIC_FUNC_ADD(long, u_long);
779 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
781 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
782 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
783 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
784 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
785 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
786 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
787 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
789 ASAN_ATOMIC_FUNC_SET(8, uint8_t);
790 ASAN_ATOMIC_FUNC_SET(16, uint16_t);
791 ASAN_ATOMIC_FUNC_SET(32, uint32_t);
792 ASAN_ATOMIC_FUNC_SET(64, uint64_t);
793 ASAN_ATOMIC_FUNC_SET(int, u_int);
794 ASAN_ATOMIC_FUNC_SET(long, u_long);
795 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
797 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
798 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
799 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
800 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
801 ASAN_ATOMIC_FUNC_CLEAR(int, u_int);
802 ASAN_ATOMIC_FUNC_CLEAR(long, u_long);
803 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
805 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
806 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
807 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int);
808 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long);
810 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
811 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
812 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
813 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
814 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
816 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
817 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
818 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
819 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
821 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
822 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
823 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
824 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
826 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t);
827 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t);
828 ASAN_ATOMIC_FUNC_SWAP(int, u_int);
829 ASAN_ATOMIC_FUNC_SWAP(long, u_long);
830 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
832 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
833 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
834 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
835 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
836 ASAN_ATOMIC_FUNC_CMPSET(int, u_int);
837 ASAN_ATOMIC_FUNC_CMPSET(long, u_long);
838 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
840 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
841 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
842 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
843 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
844 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int);
845 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long);
846 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
848 _ASAN_ATOMIC_FUNC_LOAD(bool, bool);
849 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t);
850 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t);
851 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t);
852 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t);
853 ASAN_ATOMIC_FUNC_LOAD(char, u_char);
854 ASAN_ATOMIC_FUNC_LOAD(short, u_short);
855 ASAN_ATOMIC_FUNC_LOAD(int, u_int);
856 ASAN_ATOMIC_FUNC_LOAD(long, u_long);
857 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
859 _ASAN_ATOMIC_FUNC_STORE(bool, bool);
860 ASAN_ATOMIC_FUNC_STORE(8, uint8_t);
861 ASAN_ATOMIC_FUNC_STORE(16, uint16_t);
862 ASAN_ATOMIC_FUNC_STORE(32, uint32_t);
863 ASAN_ATOMIC_FUNC_STORE(64, uint64_t);
864 ASAN_ATOMIC_FUNC_STORE(char, u_char);
865 ASAN_ATOMIC_FUNC_STORE(short, u_short);
866 ASAN_ATOMIC_FUNC_STORE(int, u_int);
867 ASAN_ATOMIC_FUNC_STORE(long, u_long);
868 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
870 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq);
871 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel);
872 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
873 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
876 kasan_atomic_interrupt_fence(void)
880 /* -------------------------------------------------------------------------- */
883 #include <machine/bus.h>
884 #include <sys/bus_san.h>
887 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
888 int flags, bus_space_handle_t *handlep)
890 return (bus_space_map(tag, hnd, size, flags, handlep));
894 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
897 bus_space_unmap(tag, hnd, size);
901 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
902 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
904 return (bus_space_subregion(tag, hnd, offset, size, handlep));
908 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
911 bus_space_free(tag, hnd, size);
915 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
916 bus_size_t offset, bus_size_t size, int flags)
918 bus_space_barrier(tag, hnd, offset, size, flags);
921 #define ASAN_BUS_READ_FUNC(func, width, type) \
922 type kasan_bus_space_read##func##_##width(bus_space_tag_t tag, \
923 bus_space_handle_t hnd, bus_size_t offset) \
925 return (bus_space_read##func##_##width(tag, hnd, \
929 #define ASAN_BUS_READ_PTR_FUNC(func, width, type) \
930 void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
931 bus_space_handle_t hnd, bus_size_t size, type *buf, \
934 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
935 false, __RET_ADDR); \
936 bus_space_read_##func##_##width(tag, hnd, size, buf, \
940 ASAN_BUS_READ_FUNC(, 1, uint8_t)
941 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t)
942 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
943 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
944 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
945 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
947 ASAN_BUS_READ_FUNC(, 2, uint16_t)
948 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t)
949 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
950 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
951 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
952 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
954 ASAN_BUS_READ_FUNC(, 4, uint32_t)
955 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t)
956 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
957 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
958 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
959 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
961 ASAN_BUS_READ_FUNC(, 8, uint64_t)
962 #if defined(__aarch64__)
963 ASAN_BUS_READ_FUNC(_stream, 8, uint64_t)
964 ASAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
965 ASAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
966 ASAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
967 ASAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
970 #define ASAN_BUS_WRITE_FUNC(func, width, type) \
971 void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \
972 bus_space_handle_t hnd, bus_size_t offset, type value) \
974 bus_space_write##func##_##width(tag, hnd, offset, value);\
977 #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type) \
978 void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
979 bus_space_handle_t hnd, bus_size_t size, const type *buf, \
982 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
984 bus_space_write_##func##_##width(tag, hnd, size, buf, \
988 ASAN_BUS_WRITE_FUNC(, 1, uint8_t)
989 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
990 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
991 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
992 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
993 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
995 ASAN_BUS_WRITE_FUNC(, 2, uint16_t)
996 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
997 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
998 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
999 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1000 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1002 ASAN_BUS_WRITE_FUNC(, 4, uint32_t)
1003 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1004 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1005 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1006 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1007 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1009 ASAN_BUS_WRITE_FUNC(, 8, uint64_t)
1011 #define ASAN_BUS_SET_FUNC(func, width, type) \
1012 void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
1013 bus_space_handle_t hnd, bus_size_t offset, type value, \
1016 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1020 ASAN_BUS_SET_FUNC(multi, 1, uint8_t)
1021 ASAN_BUS_SET_FUNC(region, 1, uint8_t)
1022 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1023 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1025 ASAN_BUS_SET_FUNC(multi, 2, uint16_t)
1026 ASAN_BUS_SET_FUNC(region, 2, uint16_t)
1027 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1028 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1030 ASAN_BUS_SET_FUNC(multi, 4, uint32_t)
1031 ASAN_BUS_SET_FUNC(region, 4, uint32_t)
1032 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1033 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1035 #define ASAN_BUS_PEEK_FUNC(width, type) \
1036 int kasan_bus_space_peek_##width(bus_space_tag_t tag, \
1037 bus_space_handle_t hnd, bus_size_t offset, type *valuep) \
1039 return (bus_space_peek_##width(tag, hnd, offset, \
1043 ASAN_BUS_PEEK_FUNC(1, uint8_t)
1044 ASAN_BUS_PEEK_FUNC(2, uint16_t)
1045 ASAN_BUS_PEEK_FUNC(4, uint32_t)
1046 ASAN_BUS_PEEK_FUNC(8, uint64_t)
1048 #define ASAN_BUS_POKE_FUNC(width, type) \
1049 int kasan_bus_space_poke_##width(bus_space_tag_t tag, \
1050 bus_space_handle_t hnd, bus_size_t offset, type value) \
1052 return (bus_space_poke_##width(tag, hnd, offset, \
1056 ASAN_BUS_POKE_FUNC(1, uint8_t)
1057 ASAN_BUS_POKE_FUNC(2, uint16_t)
1058 ASAN_BUS_POKE_FUNC(4, uint32_t)
1059 ASAN_BUS_POKE_FUNC(8, uint64_t)
1061 /* -------------------------------------------------------------------------- */
1063 void __asan_register_globals(struct __asan_global *, size_t);
1064 void __asan_unregister_globals(struct __asan_global *, size_t);
1067 __asan_register_globals(struct __asan_global *globals, size_t n)
1071 for (i = 0; i < n; i++) {
1072 kasan_mark(globals[i].beg, globals[i].size,
1073 globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
1078 __asan_unregister_globals(struct __asan_global *globals, size_t n)
1082 for (i = 0; i < n; i++) {
1083 kasan_mark(globals[i].beg, globals[i].size_with_redzone,
1084 globals[i].size_with_redzone, 0);
1088 #define ASAN_LOAD_STORE(size) \
1089 void __asan_load##size(unsigned long); \
1090 void __asan_load##size(unsigned long addr) \
1092 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1094 void __asan_load##size##_noabort(unsigned long); \
1095 void __asan_load##size##_noabort(unsigned long addr) \
1097 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1099 void __asan_store##size(unsigned long); \
1100 void __asan_store##size(unsigned long addr) \
1102 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1104 void __asan_store##size##_noabort(unsigned long); \
1105 void __asan_store##size##_noabort(unsigned long addr) \
1107 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1114 ASAN_LOAD_STORE(16);
1116 void __asan_loadN(unsigned long, size_t);
1117 void __asan_loadN_noabort(unsigned long, size_t);
1118 void __asan_storeN(unsigned long, size_t);
1119 void __asan_storeN_noabort(unsigned long, size_t);
1120 void __asan_handle_no_return(void);
1123 __asan_loadN(unsigned long addr, size_t size)
1125 kasan_shadow_check(addr, size, false, __RET_ADDR);
1129 __asan_loadN_noabort(unsigned long addr, size_t size)
1131 kasan_shadow_check(addr, size, false, __RET_ADDR);
1135 __asan_storeN(unsigned long addr, size_t size)
1137 kasan_shadow_check(addr, size, true, __RET_ADDR);
1141 __asan_storeN_noabort(unsigned long addr, size_t size)
1143 kasan_shadow_check(addr, size, true, __RET_ADDR);
1147 __asan_handle_no_return(void)
1152 #define ASAN_SET_SHADOW(byte) \
1153 void __asan_set_shadow_##byte(void *, size_t); \
1154 void __asan_set_shadow_##byte(void *addr, size_t size) \
1156 __builtin_memset((void *)addr, 0x##byte, size); \
1159 ASAN_SET_SHADOW(00);
1160 ASAN_SET_SHADOW(f1);
1161 ASAN_SET_SHADOW(f2);
1162 ASAN_SET_SHADOW(f3);
1163 ASAN_SET_SHADOW(f5);
1164 ASAN_SET_SHADOW(f8);
1166 void __asan_poison_stack_memory(const void *, size_t);
1167 void __asan_unpoison_stack_memory(const void *, size_t);
1170 __asan_poison_stack_memory(const void *addr, size_t size)
1172 size = roundup(size, KASAN_SHADOW_SCALE);
1173 kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1177 __asan_unpoison_stack_memory(const void *addr, size_t size)
1179 kasan_shadow_Nbyte_markvalid(addr, size);
1182 void __asan_alloca_poison(const void *, size_t);
1183 void __asan_allocas_unpoison(const void *, const void *);
1186 __asan_alloca_poison(const void *addr, size_t size)
1190 KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0,
1191 ("%s: invalid address %p", __func__, addr));
1193 l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1194 r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1196 kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1197 kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1199 kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1203 __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1207 if (__predict_false(!stkbegin))
1209 if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1211 size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1213 kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1216 void __asan_poison_memory_region(const void *addr, size_t size);
1217 void __asan_unpoison_memory_region(const void *addr, size_t size);
1220 __asan_poison_memory_region(const void *addr, size_t size)
1225 __asan_unpoison_memory_region(const void *addr, size_t size)