1 /* $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $ */
4 * Copyright (c) 2019 The NetBSD Foundation, Inc.
6 * Copyright (c) 2019 Andrew Turner
8 * This code is derived from software contributed to The NetBSD Foundation
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
40 #include <sys/param.h>
41 #include <sys/kernel.h>
48 #include <sys/systm.h>
51 #include <ddb/db_sym.h>
73 static csan_cpu_t kcsan_cpus[MAXCPU];
74 static bool kcsan_enabled __read_mostly;
76 #define __RET_ADDR (uintptr_t)__builtin_return_address(0)
78 #define KCSAN_NACCESSES 1024
79 #define KCSAN_DELAY 10 /* 10 microseconds */
81 /* -------------------------------------------------------------------------- */
84 #include <machine/csan.h>
86 /* -------------------------------------------------------------------------- */
89 kcsan_enable(void *dummy __unused)
92 printf("Enabling KCSCAN, expect reduced performance.\n");
95 SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL);
98 kcsan_cpu_init(u_int cpu)
100 kcsan_cpus[cpu].inited = true;
103 /* -------------------------------------------------------------------------- */
106 kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu)
108 const char *newsym, *oldsym;
113 sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset);
114 db_symbol_values(sym, &newsym, NULL);
116 sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset);
117 db_symbol_values(sym, &oldsym, NULL);
122 REPORT("CSan: Racy Access "
123 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] "
124 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n",
126 (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"),
127 (void *)new->addr, new->size, (void *)new->pc, newsym,
129 (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"),
130 (void *)old->addr, old->size, (void *)old->pc, oldsym);
135 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old)
137 if (new->write && !new->atomic)
139 if (old->write && !old->atomic)
145 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc)
147 csan_cell_t old, new;
152 if (__predict_false(!kcsan_enabled))
154 if (__predict_false(kcsan_md_unsupported((vm_offset_t)addr)))
156 if (__predict_false(panicstr != NULL))
166 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old));
168 if (old.addr + old.size <= new.addr)
170 if (new.addr + new.size <= old.addr)
172 if (__predict_true(!old.write && !new.write))
174 if (__predict_true(kcsan_access_is_atomic(&new, &old)))
177 kcsan_report(&new, PCPU_GET(cpuid), &old, i);
181 if (__predict_false(!kcsan_md_is_avail()))
184 kcsan_md_disable_intrs(&intr);
186 cpu = &kcsan_cpus[PCPU_GET(cpuid)];
187 if (__predict_false(!cpu->inited))
189 cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES;
190 if (__predict_true(cpu->cnt != 0))
193 __builtin_memcpy(&cpu->cell, &new, sizeof(new));
194 kcsan_md_delay(KCSAN_DELAY);
195 __builtin_memset(&cpu->cell, 0, sizeof(new));
198 kcsan_md_enable_intrs(&intr);
201 #define CSAN_READ(size) \
202 void __tsan_read##size(uintptr_t); \
203 void __tsan_read##size(uintptr_t addr) \
205 kcsan_access(addr, size, false, false, __RET_ADDR); \
207 void __tsan_unaligned_read##size(uintptr_t); \
208 void __tsan_unaligned_read##size(uintptr_t addr) \
210 kcsan_access(addr, size, false, false, __RET_ADDR); \
219 #define CSAN_WRITE(size) \
220 void __tsan_write##size(uintptr_t); \
221 void __tsan_write##size(uintptr_t addr) \
223 kcsan_access(addr, size, true, false, __RET_ADDR); \
225 void __tsan_unaligned_write##size(uintptr_t); \
226 void __tsan_unaligned_write##size(uintptr_t addr) \
228 kcsan_access(addr, size, true, false, __RET_ADDR); \
237 void __tsan_read_range(uintptr_t, size_t);
238 void __tsan_write_range(uintptr_t, size_t);
241 __tsan_read_range(uintptr_t addr, size_t size)
243 kcsan_access(addr, size, false, false, __RET_ADDR);
247 __tsan_write_range(uintptr_t addr, size_t size)
249 kcsan_access(addr, size, true, false, __RET_ADDR);
252 void __tsan_init(void);
253 void __tsan_func_entry(void *);
254 void __tsan_func_exit(void);
262 __tsan_func_entry(void *call_pc)
267 __tsan_func_exit(void)
271 /* -------------------------------------------------------------------------- */
274 kcsan_memcpy(void *dst, const void *src, size_t len)
276 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
277 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
278 return __builtin_memcpy(dst, src, len);
282 kcsan_memcmp(const void *b1, const void *b2, size_t len)
284 kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR);
285 kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR);
286 return __builtin_memcmp(b1, b2, len);
290 kcsan_memset(void *b, int c, size_t len)
292 kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR);
293 return __builtin_memset(b, c, len);
297 kcsan_memmove(void *dst, const void *src, size_t len)
299 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
300 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
301 return __builtin_memmove(dst, src, len);
305 kcsan_strcpy(char *dst, const char *src)
310 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR);
311 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR);
322 kcsan_strcmp(const char *s1, const char *s2)
325 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR);
326 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR);
334 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
338 kcsan_strlen(const char *str)
344 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR);
355 #undef copyin_nofault
358 #undef copyout_nofault
361 kcsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done)
363 kcsan_access((uintptr_t)kdaddr, len, true, false, __RET_ADDR);
364 return copystr(kfaddr, kdaddr, len, done);
368 kcsan_copyin(const void *uaddr, void *kaddr, size_t len)
370 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
371 return copyin(uaddr, kaddr, len);
375 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
377 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
378 return copyinstr(uaddr, kaddr, len, done);
382 kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
384 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
385 return copyout(kaddr, uaddr, len);
388 /* -------------------------------------------------------------------------- */
390 #include <machine/atomic.h>
391 #include <sys/_cscan_atomic.h>
393 #define _CSAN_ATOMIC_FUNC_ADD(name, type) \
394 void kcsan_atomic_add_##name(volatile type *ptr, type val) \
396 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
398 atomic_add_##name(ptr, val); \
401 #define CSAN_ATOMIC_FUNC_ADD(name, type) \
402 _CSAN_ATOMIC_FUNC_ADD(name, type) \
403 _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \
404 _CSAN_ATOMIC_FUNC_ADD(rel_##name, type)
406 #define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
407 void kcsan_atomic_clear_##name(volatile type *ptr, type val) \
409 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
411 atomic_clear_##name(ptr, val); \
414 #define CSAN_ATOMIC_FUNC_CLEAR(name, type) \
415 _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
416 _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
417 _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
419 #define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
420 int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \
423 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
425 return (atomic_cmpset_##name(ptr, val1, val2)); \
428 #define CSAN_ATOMIC_FUNC_CMPSET(name, type) \
429 _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
430 _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
431 _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
433 #define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
434 int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \
437 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
439 return (atomic_fcmpset_##name(ptr, val1, val2)); \
442 #define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
443 _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
444 _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
445 _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
447 #define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \
448 type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
450 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
452 return (atomic_fetchadd_##name(ptr, val)); \
455 #define _CSAN_ATOMIC_FUNC_LOAD(name, type) \
456 type kcsan_atomic_load_##name(volatile type *ptr) \
458 kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \
460 return (atomic_load_##name(ptr)); \
463 #define CSAN_ATOMIC_FUNC_LOAD(name, type) \
464 _CSAN_ATOMIC_FUNC_LOAD(name, type) \
465 _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \
467 #define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
468 type kcsan_atomic_readandclear_##name(volatile type *ptr) \
470 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
472 return (atomic_readandclear_##name(ptr)); \
475 #define _CSAN_ATOMIC_FUNC_SET(name, type) \
476 void kcsan_atomic_set_##name(volatile type *ptr, type val) \
478 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
480 atomic_set_##name(ptr, val); \
483 #define CSAN_ATOMIC_FUNC_SET(name, type) \
484 _CSAN_ATOMIC_FUNC_SET(name, type) \
485 _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \
486 _CSAN_ATOMIC_FUNC_SET(rel_##name, type)
488 #define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
489 void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \
491 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
493 atomic_subtract_##name(ptr, val); \
497 #define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
498 _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
499 _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
500 _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
502 #define _CSAN_ATOMIC_FUNC_STORE(name, type) \
503 void kcsan_atomic_store_##name(volatile type *ptr, type val) \
505 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
507 atomic_store_##name(ptr, val); \
510 #define CSAN_ATOMIC_FUNC_STORE(name, type) \
511 _CSAN_ATOMIC_FUNC_STORE(name, type) \
512 _CSAN_ATOMIC_FUNC_STORE(rel_##name, type)
514 #define CSAN_ATOMIC_FUNC_SWAP(name, type) \
515 type kcsan_atomic_swap_##name(volatile type *ptr, type val) \
517 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
519 return(atomic_swap_##name(ptr, val)); \
522 #define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
523 int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \
525 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
527 return(atomic_testandclear_##name(ptr, val)); \
530 #define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \
531 int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \
533 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
535 return (atomic_testandset_##name(ptr, val)); \
539 CSAN_ATOMIC_FUNC_ADD(8, uint8_t)
540 CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t)
541 CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t)
542 CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t)
543 _CSAN_ATOMIC_FUNC_LOAD(8, uint8_t)
544 CSAN_ATOMIC_FUNC_SET(8, uint8_t)
545 CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t)
546 _CSAN_ATOMIC_FUNC_STORE(8, uint8_t)
548 CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t)
549 CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t)
550 CSAN_ATOMIC_FUNC_SWAP(8, uint8_t)
551 CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t)
552 CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t)
555 CSAN_ATOMIC_FUNC_ADD(16, uint16_t)
556 CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t)
557 CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t)
558 CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t)
559 #if defined(__aarch64__)
560 _CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
562 CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
564 CSAN_ATOMIC_FUNC_SET(16, uint16_t)
565 CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t)
566 _CSAN_ATOMIC_FUNC_STORE(16, uint16_t)
568 CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t)
569 CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t)
570 CSAN_ATOMIC_FUNC_SWAP(16, uint16_t)
571 CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t)
572 CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t)
575 CSAN_ATOMIC_FUNC_ADD(32, uint32_t)
576 CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t)
577 CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t)
578 CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t)
579 CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t)
580 CSAN_ATOMIC_FUNC_LOAD(32, uint32_t)
581 CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t)
582 CSAN_ATOMIC_FUNC_SET(32, uint32_t)
583 CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t)
584 CSAN_ATOMIC_FUNC_STORE(32, uint32_t)
585 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t)
586 #if !defined(__aarch64__)
587 CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t)
588 CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t)
591 CSAN_ATOMIC_FUNC_ADD(64, uint64_t)
592 CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t)
593 CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t)
594 CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t)
595 CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t)
596 CSAN_ATOMIC_FUNC_LOAD(64, uint64_t)
597 CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t)
598 CSAN_ATOMIC_FUNC_SET(64, uint64_t)
599 CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t)
600 CSAN_ATOMIC_FUNC_STORE(64, uint64_t)
601 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t)
602 #if !defined(__aarch64__)
603 CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t)
604 CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t)
607 CSAN_ATOMIC_FUNC_ADD(int, u_int)
608 CSAN_ATOMIC_FUNC_CLEAR(int, u_int)
609 CSAN_ATOMIC_FUNC_CMPSET(int, u_int)
610 CSAN_ATOMIC_FUNC_FCMPSET(int, u_int)
611 CSAN_ATOMIC_FUNC_FETCHADD(int, u_int)
612 CSAN_ATOMIC_FUNC_LOAD(int, u_int)
613 CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int)
614 CSAN_ATOMIC_FUNC_SET(int, u_int)
615 CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int)
616 CSAN_ATOMIC_FUNC_STORE(int, u_int)
617 CSAN_ATOMIC_FUNC_SWAP(int, u_int)
618 #if !defined(__aarch64__)
619 CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int)
620 CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int)
623 CSAN_ATOMIC_FUNC_ADD(long, u_long)
624 CSAN_ATOMIC_FUNC_CLEAR(long, u_long)
625 CSAN_ATOMIC_FUNC_CMPSET(long, u_long)
626 CSAN_ATOMIC_FUNC_FCMPSET(long, u_long)
627 CSAN_ATOMIC_FUNC_FETCHADD(long, u_long)
628 CSAN_ATOMIC_FUNC_LOAD(long, u_long)
629 CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long)
630 CSAN_ATOMIC_FUNC_SET(long, u_long)
631 CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long)
632 CSAN_ATOMIC_FUNC_STORE(long, u_long)
633 CSAN_ATOMIC_FUNC_SWAP(long, u_long)
634 #if !defined(__aarch64__)
635 CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long)
636 CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long)
639 CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t)
640 CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t)
641 CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t)
642 CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t)
643 #if !defined(__amd64__)
644 CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t)
646 CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t)
647 CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t)
648 CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t)
649 CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t)
650 CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t)
651 CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t)
653 CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t)
654 CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t)
657 #define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \
658 void kcsan_atomic_thread_fence_##name(void) \
660 atomic_thread_fence_##name(); \
664 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq)
665 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel)
666 CSAN_ATOMIC_FUNC_THREAD_FENCE(rel)
667 CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst)
669 /* -------------------------------------------------------------------------- */
672 #include <machine/bus.h>
673 #include <sys/_cscan_bus.h>
676 kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
677 int flags, bus_space_handle_t *handlep)
680 return (bus_space_map(tag, hnd, size, flags, handlep));
684 kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
688 bus_space_unmap(tag, hnd, size);
692 kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
693 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
696 return (bus_space_subregion(tag, hnd, offset, size, handlep));
699 #if !defined(__amd64__)
701 kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start,
702 bus_addr_t reg_end, bus_size_t size, bus_size_t alignment,
703 bus_size_t boundary, int flags, bus_addr_t *addrp,
704 bus_space_handle_t *handlep)
707 return (bus_space_alloc(tag, reg_start, reg_end, size, alignment,
708 boundary, flags, addrp, handlep));
713 kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
717 bus_space_free(tag, hnd, size);
721 kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
722 bus_size_t offset, bus_size_t size, int flags)
725 bus_space_barrier(tag, hnd, offset, size, flags);
728 #define CSAN_BUS_READ_FUNC(func, width, type) \
729 type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \
730 bus_space_handle_t hnd, bus_size_t offset) \
732 return (bus_space_read##func##_##width(tag, hnd, \
736 #define CSAN_BUS_READ_PTR_FUNC(func, width, type) \
737 void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
738 bus_space_handle_t hnd, bus_size_t size, type *buf, \
741 kcsan_access((uintptr_t)buf, sizeof(type) * count, \
742 false, false, __RET_ADDR); \
743 bus_space_read_##func##_##width(tag, hnd, size, buf, \
747 CSAN_BUS_READ_FUNC(, 1, uint8_t)
748 CSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
749 CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
750 CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
751 CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
752 CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
754 CSAN_BUS_READ_FUNC(, 2, uint16_t)
755 CSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
756 CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
757 CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
758 CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
759 CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
761 CSAN_BUS_READ_FUNC(, 4, uint32_t)
762 CSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
763 CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
764 CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
765 CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
766 CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
768 CSAN_BUS_READ_FUNC(, 8, uint64_t)
769 #if defined(__aarch64__)
770 CSAN_BUS_READ_FUNC(_stream, 8, uint64_t)
771 CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
772 CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
773 CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
774 CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
777 #define CSAN_BUS_WRITE_FUNC(func, width, type) \
778 void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
779 bus_space_handle_t hnd, bus_size_t offset, type value) \
781 bus_space_write##func##_##width(tag, hnd, offset, value); \
784 #define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \
785 void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \
786 bus_space_handle_t hnd, bus_size_t size, const type *buf, \
789 kcsan_access((uintptr_t)buf, sizeof(type) * count, \
790 true, false, __RET_ADDR); \
791 bus_space_write_##func##_##width(tag, hnd, size, buf, \
795 CSAN_BUS_WRITE_FUNC(, 1, uint8_t)
796 CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
797 CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
798 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
799 CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
800 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
802 CSAN_BUS_WRITE_FUNC(, 2, uint16_t)
803 CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
804 CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
805 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
806 CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
807 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
809 CSAN_BUS_WRITE_FUNC(, 4, uint32_t)
810 CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
811 CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
812 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
813 CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
814 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
816 CSAN_BUS_WRITE_FUNC(, 8, uint64_t)
817 #if defined(__aarch64__)
818 CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t)
819 CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t)
820 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t)
821 CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t)
822 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t)
825 #define CSAN_BUS_SET_FUNC(func, width, type) \
826 void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
827 bus_space_handle_t hnd, bus_size_t offset, type value, \
830 bus_space_set_##func##_##width(tag, hnd, offset, value, \
834 CSAN_BUS_SET_FUNC(multi, 1, uint8_t)
835 CSAN_BUS_SET_FUNC(region, 1, uint8_t)
836 #if !defined(__aarch64__)
837 CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
838 CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
841 CSAN_BUS_SET_FUNC(multi, 2, uint16_t)
842 CSAN_BUS_SET_FUNC(region, 2, uint16_t)
843 #if !defined(__aarch64__)
844 CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
845 CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
848 CSAN_BUS_SET_FUNC(multi, 4, uint32_t)
849 CSAN_BUS_SET_FUNC(region, 4, uint32_t)
850 #if !defined(__aarch64__)
851 CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
852 CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
855 #if !defined(__amd64__)
856 CSAN_BUS_SET_FUNC(multi, 8, uint64_t)
857 CSAN_BUS_SET_FUNC(region, 8, uint64_t)
858 #if !defined(__aarch64__)
859 CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t)
860 CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t)