1 /* $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $ */
4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
7 * This code is part of the KASAN subsystem of the NetBSD kernel.
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
36 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $");
39 #include <sys/param.h>
40 #include <sys/systm.h>
42 #include <sys/kernel.h>
43 #include <sys/stack.h>
44 #include <sys/sysctl.h>
46 #include <machine/asan.h>
47 #include <machine/bus.h>
49 /* ASAN constants. Part of the compiler ABI. */
50 #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE - 1)
51 #define KASAN_ALLOCA_SCALE_SIZE 32
53 /* ASAN ABI version. */
54 #if defined(__clang__) && (__clang_major__ - 0 >= 6)
55 #define ASAN_ABI_VERSION 8
56 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
57 #define ASAN_ABI_VERSION 8
58 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
59 #define ASAN_ABI_VERSION 6
61 #error "Unsupported compiler version"
64 #define __RET_ADDR (unsigned long)__builtin_return_address(0)
66 /* Global variable descriptor. Part of the compiler ABI. */
67 struct __asan_global_source_location {
73 struct __asan_global {
74 const void *beg; /* address of the global variable */
75 size_t size; /* size of the global variable */
76 size_t size_with_redzone; /* size with the redzone */
77 const void *name; /* name of the variable */
78 const void *module_name; /* name of the module where the var is declared */
79 unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
80 struct __asan_global_source_location *location;
81 #if ASAN_ABI_VERSION >= 7
82 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */
86 FEATURE(kasan, "Kernel address sanitizer");
88 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
91 static int panic_on_violation = 1;
92 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN,
93 &panic_on_violation, 0,
94 "Panic if an invalid access is detected");
96 static bool kasan_enabled __read_mostly = false;
98 /* -------------------------------------------------------------------------- */
101 kasan_shadow_map(vm_offset_t addr, size_t size)
103 size_t sz, npages, i;
104 vm_offset_t sva, eva;
106 KASSERT(addr % KASAN_SHADOW_SCALE == 0,
107 ("%s: invalid address %#lx", __func__, addr));
109 sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE;
111 sva = kasan_md_addr_to_shad(addr);
112 eva = kasan_md_addr_to_shad(addr) + sz;
114 sva = rounddown(sva, PAGE_SIZE);
115 eva = roundup(eva, PAGE_SIZE);
117 npages = (eva - sva) / PAGE_SIZE;
119 KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS,
120 ("%s: invalid address range %#lx-%#lx", __func__, sva, eva));
122 for (i = 0; i < npages; i++)
123 pmap_kasan_enter(sva + ptoa(i));
132 TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled);
136 /* MD initialization. */
139 /* Now officially enabled. */
140 kasan_enabled = true;
143 static inline const char *
144 kasan_code_name(uint8_t code)
147 case KASAN_GENERIC_REDZONE:
148 return "GenericRedZone";
149 case KASAN_MALLOC_REDZONE:
150 return "MallocRedZone";
151 case KASAN_KMEM_REDZONE:
152 return "KmemRedZone";
153 case KASAN_UMA_FREED:
154 return "UMAUseAfterFree";
155 case KASAN_KSTACK_FREED:
156 return "KernelStack";
157 case KASAN_EXEC_ARGS_FREED:
160 return "RedZonePartial";
161 case KASAN_STACK_LEFT:
163 case KASAN_STACK_MID:
164 return "StackMiddle";
165 case KASAN_STACK_RIGHT:
167 case KASAN_USE_AFTER_RET:
168 return "UseAfterRet";
169 case KASAN_USE_AFTER_SCOPE:
170 return "UseAfterScope";
176 #define REPORT(f, ...) do { \
177 if (panic_on_violation) { \
178 kasan_enabled = false; \
179 panic(f, __VA_ARGS__); \
184 printf(f "\n", __VA_ARGS__); \
185 stack_print_ddb(&st); \
190 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
193 REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)",
194 size, (write ? "write" : "read"), addr, kasan_code_name(code),
198 static __always_inline void
199 kasan_shadow_1byte_markvalid(unsigned long addr)
201 int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
202 int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
207 static __always_inline void
208 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
212 for (i = 0; i < size; i++) {
213 kasan_shadow_1byte_markvalid((unsigned long)addr + i);
217 static __always_inline void
218 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
222 if (__predict_false(size == 0))
224 if (__predict_false(kasan_md_unsupported((vm_offset_t)addr)))
227 KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
228 ("%s: invalid address %p", __func__, addr));
229 KASSERT(size % KASAN_SHADOW_SCALE == 0,
230 ("%s: invalid size %zu", __func__, size));
232 shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr);
233 size = size >> KASAN_SHADOW_SCALE_SHIFT;
235 __builtin_memset(shad, code, size);
239 * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
240 * and the rest as invalid. There are generally two use cases:
242 * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
243 * the redzone at the end of the buffer as invalid. If the entire is to be
244 * marked invalid, origsize will be 0.
246 * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
249 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
254 if (__predict_false(!kasan_enabled))
257 if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS &&
258 (vm_offset_t)addr < DMAP_MAX_ADDRESS)
261 KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS &&
262 (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS,
263 ("%s: invalid address %p", __func__, addr));
264 KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
265 ("%s: invalid address %p", __func__, addr));
266 redz = redzsize - roundup(size, KASAN_SHADOW_SCALE);
267 KASSERT(redz % KASAN_SHADOW_SCALE == 0,
268 ("%s: invalid size %zu", __func__, redz));
269 shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr);
271 /* Chunks of 8 bytes, valid. */
272 n = size / KASAN_SHADOW_SCALE;
273 for (i = 0; i < n; i++) {
277 /* Possibly one chunk, mid. */
278 if ((size & KASAN_SHADOW_MASK) != 0) {
279 *shad++ = (size & KASAN_SHADOW_MASK);
282 /* Chunks of 8 bytes, invalid. */
283 n = redz / KASAN_SHADOW_SCALE;
284 for (i = 0; i < n; i++) {
289 /* -------------------------------------------------------------------------- */
291 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \
292 (addr >> KASAN_SHADOW_SCALE_SHIFT) != \
293 ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
295 static __always_inline bool
296 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
298 int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
299 int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
301 if (__predict_true(*byte == 0 || last <= *byte)) {
308 static __always_inline bool
309 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
313 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
314 return (kasan_shadow_1byte_isvalid(addr, code) &&
315 kasan_shadow_1byte_isvalid(addr+1, code));
318 byte = (int8_t *)kasan_md_addr_to_shad(addr);
319 last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
321 if (__predict_true(*byte == 0 || last <= *byte)) {
328 static __always_inline bool
329 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
333 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
334 return (kasan_shadow_2byte_isvalid(addr, code) &&
335 kasan_shadow_2byte_isvalid(addr+2, code));
338 byte = (int8_t *)kasan_md_addr_to_shad(addr);
339 last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
341 if (__predict_true(*byte == 0 || last <= *byte)) {
348 static __always_inline bool
349 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
353 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
354 return (kasan_shadow_4byte_isvalid(addr, code) &&
355 kasan_shadow_4byte_isvalid(addr+4, code));
358 byte = (int8_t *)kasan_md_addr_to_shad(addr);
359 last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
361 if (__predict_true(*byte == 0 || last <= *byte)) {
368 static __always_inline bool
369 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
373 for (i = 0; i < size; i++) {
374 if (!kasan_shadow_1byte_isvalid(addr+i, code))
381 static __always_inline void
382 kasan_shadow_check(unsigned long addr, size_t size, bool write,
383 unsigned long retaddr)
388 if (__predict_false(!kasan_enabled))
390 if (__predict_false(size == 0))
392 if (__predict_false(kasan_md_unsupported(addr)))
394 if (KERNEL_PANICKED())
397 if (__builtin_constant_p(size)) {
400 valid = kasan_shadow_1byte_isvalid(addr, &code);
403 valid = kasan_shadow_2byte_isvalid(addr, &code);
406 valid = kasan_shadow_4byte_isvalid(addr, &code);
409 valid = kasan_shadow_8byte_isvalid(addr, &code);
412 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
416 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
419 if (__predict_false(!valid)) {
420 kasan_report(addr, size, write, retaddr, code);
424 /* -------------------------------------------------------------------------- */
427 kasan_memcpy(void *dst, const void *src, size_t len)
429 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
430 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
431 return (__builtin_memcpy(dst, src, len));
435 kasan_memcmp(const void *b1, const void *b2, size_t len)
437 kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
438 kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
439 return (__builtin_memcmp(b1, b2, len));
443 kasan_memset(void *b, int c, size_t len)
445 kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
446 return (__builtin_memset(b, c, len));
450 kasan_memmove(void *dst, const void *src, size_t len)
452 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
453 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
454 return (__builtin_memmove(dst, src, len));
458 kasan_strlen(const char *str)
464 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
474 kasan_strcpy(char *dst, const char *src)
479 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
480 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
491 kasan_strcmp(const char *s1, const char *s2)
494 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
495 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
503 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
507 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
509 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
510 return (copyin(uaddr, kaddr, len));
514 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
516 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
517 return (copyinstr(uaddr, kaddr, len, done));
521 kasan_copyout(const void *kaddr, void *uaddr, size_t len)
523 kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
524 return (copyout(kaddr, uaddr, len));
527 /* -------------------------------------------------------------------------- */
529 #include <machine/atomic.h>
530 #include <sys/atomic_san.h>
532 #define _ASAN_ATOMIC_FUNC_ADD(name, type) \
533 void kasan_atomic_add_##name(volatile type *ptr, type val) \
535 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
537 atomic_add_##name(ptr, val); \
540 #define ASAN_ATOMIC_FUNC_ADD(name, type) \
541 _ASAN_ATOMIC_FUNC_ADD(name, type) \
542 _ASAN_ATOMIC_FUNC_ADD(acq_##name, type) \
543 _ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
545 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
546 void kasan_atomic_subtract_##name(volatile type *ptr, type val) \
548 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
550 atomic_subtract_##name(ptr, val); \
553 #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
554 _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
555 _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
556 _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
558 #define _ASAN_ATOMIC_FUNC_SET(name, type) \
559 void kasan_atomic_set_##name(volatile type *ptr, type val) \
561 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
563 atomic_set_##name(ptr, val); \
566 #define ASAN_ATOMIC_FUNC_SET(name, type) \
567 _ASAN_ATOMIC_FUNC_SET(name, type) \
568 _ASAN_ATOMIC_FUNC_SET(acq_##name, type) \
569 _ASAN_ATOMIC_FUNC_SET(rel_##name, type)
571 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type) \
572 void kasan_atomic_clear_##name(volatile type *ptr, type val) \
574 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
576 atomic_clear_##name(ptr, val); \
579 #define ASAN_ATOMIC_FUNC_CLEAR(name, type) \
580 _ASAN_ATOMIC_FUNC_CLEAR(name, type) \
581 _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
582 _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
584 #define ASAN_ATOMIC_FUNC_FETCHADD(name, type) \
585 type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \
587 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
589 return (atomic_fetchadd_##name(ptr, val)); \
592 #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
593 type kasan_atomic_readandclear_##name(volatile type *ptr) \
595 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
597 return (atomic_readandclear_##name(ptr)); \
600 #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
601 int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
603 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
605 return (atomic_testandclear_##name(ptr, v)); \
608 #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type) \
609 int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
611 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
613 return (atomic_testandset_##name(ptr, v)); \
616 #define ASAN_ATOMIC_FUNC_SWAP(name, type) \
617 type kasan_atomic_swap_##name(volatile type *ptr, type val) \
619 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
621 return (atomic_swap_##name(ptr, val)); \
624 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type) \
625 int kasan_atomic_cmpset_##name(volatile type *ptr, type oval, \
628 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
630 return (atomic_cmpset_##name(ptr, oval, nval)); \
633 #define ASAN_ATOMIC_FUNC_CMPSET(name, type) \
634 _ASAN_ATOMIC_FUNC_CMPSET(name, type) \
635 _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
636 _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
638 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
639 int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
642 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
644 return (atomic_fcmpset_##name(ptr, oval, nval)); \
647 #define ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
648 _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
649 _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
650 _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
652 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name) \
653 void kasan_atomic_thread_fence_##name(void) \
655 atomic_thread_fence_##name(); \
658 #define _ASAN_ATOMIC_FUNC_LOAD(name, type) \
659 type kasan_atomic_load_##name(volatile type *ptr) \
661 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
663 return (atomic_load_##name(ptr)); \
666 #define ASAN_ATOMIC_FUNC_LOAD(name, type) \
667 _ASAN_ATOMIC_FUNC_LOAD(name, type) \
668 _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
670 #define _ASAN_ATOMIC_FUNC_STORE(name, type) \
671 void kasan_atomic_store_##name(volatile type *ptr, type val) \
673 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
675 atomic_store_##name(ptr, val); \
678 #define ASAN_ATOMIC_FUNC_STORE(name, type) \
679 _ASAN_ATOMIC_FUNC_STORE(name, type) \
680 _ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
682 ASAN_ATOMIC_FUNC_ADD(8, uint8_t);
683 ASAN_ATOMIC_FUNC_ADD(16, uint16_t);
684 ASAN_ATOMIC_FUNC_ADD(32, uint32_t);
685 ASAN_ATOMIC_FUNC_ADD(64, uint64_t);
686 ASAN_ATOMIC_FUNC_ADD(int, u_int);
687 ASAN_ATOMIC_FUNC_ADD(long, u_long);
688 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
690 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
691 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
692 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
693 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
694 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
695 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
696 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
698 ASAN_ATOMIC_FUNC_SET(8, uint8_t);
699 ASAN_ATOMIC_FUNC_SET(16, uint16_t);
700 ASAN_ATOMIC_FUNC_SET(32, uint32_t);
701 ASAN_ATOMIC_FUNC_SET(64, uint64_t);
702 ASAN_ATOMIC_FUNC_SET(int, u_int);
703 ASAN_ATOMIC_FUNC_SET(long, u_long);
704 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
706 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
707 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
708 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
709 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
710 ASAN_ATOMIC_FUNC_CLEAR(int, u_int);
711 ASAN_ATOMIC_FUNC_CLEAR(long, u_long);
712 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
714 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
715 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
716 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int);
717 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long);
719 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
720 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
721 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
722 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
723 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
725 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
726 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
727 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
728 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
730 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
731 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
732 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
733 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
735 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t);
736 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t);
737 ASAN_ATOMIC_FUNC_SWAP(int, u_int);
738 ASAN_ATOMIC_FUNC_SWAP(long, u_long);
739 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
741 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
742 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
743 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
744 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
745 ASAN_ATOMIC_FUNC_CMPSET(int, u_int);
746 ASAN_ATOMIC_FUNC_CMPSET(long, u_long);
747 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
749 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
750 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
751 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
752 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
753 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int);
754 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long);
755 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
757 _ASAN_ATOMIC_FUNC_LOAD(bool, bool);
758 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t);
759 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t);
760 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t);
761 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t);
762 ASAN_ATOMIC_FUNC_LOAD(char, u_char);
763 ASAN_ATOMIC_FUNC_LOAD(short, u_short);
764 ASAN_ATOMIC_FUNC_LOAD(int, u_int);
765 ASAN_ATOMIC_FUNC_LOAD(long, u_long);
766 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
768 _ASAN_ATOMIC_FUNC_STORE(bool, bool);
769 ASAN_ATOMIC_FUNC_STORE(8, uint8_t);
770 ASAN_ATOMIC_FUNC_STORE(16, uint16_t);
771 ASAN_ATOMIC_FUNC_STORE(32, uint32_t);
772 ASAN_ATOMIC_FUNC_STORE(64, uint64_t);
773 ASAN_ATOMIC_FUNC_STORE(char, u_char);
774 ASAN_ATOMIC_FUNC_STORE(short, u_short);
775 ASAN_ATOMIC_FUNC_STORE(int, u_int);
776 ASAN_ATOMIC_FUNC_STORE(long, u_long);
777 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
779 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq);
780 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel);
781 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
782 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
785 kasan_atomic_interrupt_fence(void)
789 /* -------------------------------------------------------------------------- */
792 #include <machine/bus.h>
793 #include <sys/bus_san.h>
796 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
797 int flags, bus_space_handle_t *handlep)
799 return (bus_space_map(tag, hnd, size, flags, handlep));
803 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
806 bus_space_unmap(tag, hnd, size);
810 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
811 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
813 return (bus_space_subregion(tag, hnd, offset, size, handlep));
817 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
820 bus_space_free(tag, hnd, size);
824 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
825 bus_size_t offset, bus_size_t size, int flags)
827 bus_space_barrier(tag, hnd, offset, size, flags);
830 #define ASAN_BUS_READ_FUNC(func, width, type) \
831 type kasan_bus_space_read##func##_##width(bus_space_tag_t tag, \
832 bus_space_handle_t hnd, bus_size_t offset) \
834 return (bus_space_read##func##_##width(tag, hnd, \
838 #define ASAN_BUS_READ_PTR_FUNC(func, width, type) \
839 void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
840 bus_space_handle_t hnd, bus_size_t size, type *buf, \
843 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
844 false, __RET_ADDR); \
845 bus_space_read_##func##_##width(tag, hnd, size, buf, \
849 ASAN_BUS_READ_FUNC(, 1, uint8_t)
850 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t)
851 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
852 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
853 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
854 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
856 ASAN_BUS_READ_FUNC(, 2, uint16_t)
857 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t)
858 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
859 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
860 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
861 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
863 ASAN_BUS_READ_FUNC(, 4, uint32_t)
864 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t)
865 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
866 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
867 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
868 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
870 ASAN_BUS_READ_FUNC(, 8, uint64_t)
872 #define ASAN_BUS_WRITE_FUNC(func, width, type) \
873 void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \
874 bus_space_handle_t hnd, bus_size_t offset, type value) \
876 bus_space_write##func##_##width(tag, hnd, offset, value);\
879 #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type) \
880 void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
881 bus_space_handle_t hnd, bus_size_t size, const type *buf, \
884 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
886 bus_space_write_##func##_##width(tag, hnd, size, buf, \
890 ASAN_BUS_WRITE_FUNC(, 1, uint8_t)
891 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
892 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
893 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
894 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
895 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
897 ASAN_BUS_WRITE_FUNC(, 2, uint16_t)
898 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
899 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
900 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
901 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
902 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
904 ASAN_BUS_WRITE_FUNC(, 4, uint32_t)
905 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
906 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
907 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
908 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
909 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
911 ASAN_BUS_WRITE_FUNC(, 8, uint64_t)
913 #define ASAN_BUS_SET_FUNC(func, width, type) \
914 void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
915 bus_space_handle_t hnd, bus_size_t offset, type value, \
918 bus_space_set_##func##_##width(tag, hnd, offset, value, \
922 ASAN_BUS_SET_FUNC(multi, 1, uint8_t)
923 ASAN_BUS_SET_FUNC(region, 1, uint8_t)
924 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
925 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
927 ASAN_BUS_SET_FUNC(multi, 2, uint16_t)
928 ASAN_BUS_SET_FUNC(region, 2, uint16_t)
929 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
930 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
932 ASAN_BUS_SET_FUNC(multi, 4, uint32_t)
933 ASAN_BUS_SET_FUNC(region, 4, uint32_t)
934 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
935 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
937 #define ASAN_BUS_PEEK_FUNC(width, type) \
938 int kasan_bus_space_peek_##width(bus_space_tag_t tag, \
939 bus_space_handle_t hnd, bus_size_t offset, type *valuep) \
941 return (bus_space_peek_##width(tag, hnd, offset, \
945 ASAN_BUS_PEEK_FUNC(1, uint8_t)
946 ASAN_BUS_PEEK_FUNC(2, uint16_t)
947 ASAN_BUS_PEEK_FUNC(4, uint32_t)
948 ASAN_BUS_PEEK_FUNC(8, uint64_t)
950 #define ASAN_BUS_POKE_FUNC(width, type) \
951 int kasan_bus_space_poke_##width(bus_space_tag_t tag, \
952 bus_space_handle_t hnd, bus_size_t offset, type value) \
954 return (bus_space_poke_##width(tag, hnd, offset, \
958 ASAN_BUS_POKE_FUNC(1, uint8_t)
959 ASAN_BUS_POKE_FUNC(2, uint16_t)
960 ASAN_BUS_POKE_FUNC(4, uint32_t)
961 ASAN_BUS_POKE_FUNC(8, uint64_t)
963 /* -------------------------------------------------------------------------- */
965 void __asan_register_globals(struct __asan_global *, size_t);
966 void __asan_unregister_globals(struct __asan_global *, size_t);
969 __asan_register_globals(struct __asan_global *globals, size_t n)
973 for (i = 0; i < n; i++) {
974 kasan_mark(globals[i].beg, globals[i].size,
975 globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
980 __asan_unregister_globals(struct __asan_global *globals, size_t n)
984 for (i = 0; i < n; i++) {
985 kasan_mark(globals[i].beg, globals[i].size_with_redzone,
986 globals[i].size_with_redzone, 0);
990 #define ASAN_LOAD_STORE(size) \
991 void __asan_load##size(unsigned long); \
992 void __asan_load##size(unsigned long addr) \
994 kasan_shadow_check(addr, size, false, __RET_ADDR);\
996 void __asan_load##size##_noabort(unsigned long); \
997 void __asan_load##size##_noabort(unsigned long addr) \
999 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1001 void __asan_store##size(unsigned long); \
1002 void __asan_store##size(unsigned long addr) \
1004 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1006 void __asan_store##size##_noabort(unsigned long); \
1007 void __asan_store##size##_noabort(unsigned long addr) \
1009 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1016 ASAN_LOAD_STORE(16);
1018 void __asan_loadN(unsigned long, size_t);
1019 void __asan_loadN_noabort(unsigned long, size_t);
1020 void __asan_storeN(unsigned long, size_t);
1021 void __asan_storeN_noabort(unsigned long, size_t);
1022 void __asan_handle_no_return(void);
1025 __asan_loadN(unsigned long addr, size_t size)
1027 kasan_shadow_check(addr, size, false, __RET_ADDR);
1031 __asan_loadN_noabort(unsigned long addr, size_t size)
1033 kasan_shadow_check(addr, size, false, __RET_ADDR);
1037 __asan_storeN(unsigned long addr, size_t size)
1039 kasan_shadow_check(addr, size, true, __RET_ADDR);
1043 __asan_storeN_noabort(unsigned long addr, size_t size)
1045 kasan_shadow_check(addr, size, true, __RET_ADDR);
1049 __asan_handle_no_return(void)
1054 #define ASAN_SET_SHADOW(byte) \
1055 void __asan_set_shadow_##byte(void *, size_t); \
1056 void __asan_set_shadow_##byte(void *addr, size_t size) \
1058 __builtin_memset((void *)addr, 0x##byte, size); \
1061 ASAN_SET_SHADOW(00);
1062 ASAN_SET_SHADOW(f1);
1063 ASAN_SET_SHADOW(f2);
1064 ASAN_SET_SHADOW(f3);
1065 ASAN_SET_SHADOW(f5);
1066 ASAN_SET_SHADOW(f8);
1068 void __asan_poison_stack_memory(const void *, size_t);
1069 void __asan_unpoison_stack_memory(const void *, size_t);
1072 __asan_poison_stack_memory(const void *addr, size_t size)
1074 size = roundup(size, KASAN_SHADOW_SCALE);
1075 kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1079 __asan_unpoison_stack_memory(const void *addr, size_t size)
1081 kasan_shadow_Nbyte_markvalid(addr, size);
1084 void __asan_alloca_poison(const void *, size_t);
1085 void __asan_allocas_unpoison(const void *, const void *);
1088 __asan_alloca_poison(const void *addr, size_t size)
1092 KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0,
1093 ("%s: invalid address %p", __func__, addr));
1095 l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1096 r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1098 kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1099 kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1101 kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1105 __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1109 if (__predict_false(!stkbegin))
1111 if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1113 size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1115 kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1118 void __asan_poison_memory_region(const void *addr, size_t size);
1119 void __asan_unpoison_memory_region(const void *addr, size_t size);
1122 __asan_poison_memory_region(const void *addr, size_t size)
1127 __asan_unpoison_memory_region(const void *addr, size_t size)