sys/kern/subr_asan.c

   1 /*      $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $      */
   2
   3 /*
   4  * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
   5  * All rights reserved.
   6  *
   7  * This code is part of the KASAN subsystem of the NetBSD kernel.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  23  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  25  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  26  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 #define SAN_RUNTIME
  32
  33 #include <sys/cdefs.h>
  34 #if 0
  35 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $");
  36 #endif
  37
  38 #include <sys/param.h>
  39 #include <sys/systm.h>
  40 #include <sys/asan.h>
  41 #include <sys/kernel.h>
  42 #include <sys/stack.h>
  43 #include <sys/sysctl.h>
  44
  45 #include <machine/asan.h>
  46 #include <machine/bus.h>
  47
  48 /* ASAN constants. Part of the compiler ABI. */
  49 #define KASAN_SHADOW_MASK               (KASAN_SHADOW_SCALE - 1)
  50 #define KASAN_ALLOCA_SCALE_SIZE         32
  51
  52 /* ASAN ABI version. */
  53 #if defined(__clang__) && (__clang_major__ - 0 >= 6)
  54 #define ASAN_ABI_VERSION        8
  55 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
  56 #define ASAN_ABI_VERSION        8
  57 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
  58 #define ASAN_ABI_VERSION        6
  59 #else
  60 #error "Unsupported compiler version"
  61 #endif
  62
  63 #define __RET_ADDR      (unsigned long)__builtin_return_address(0)
  64
  65 /* Global variable descriptor. Part of the compiler ABI.  */
  66 struct __asan_global_source_location {
  67         const char *filename;
  68         int line_no;
  69         int column_no;
  70 };
  71
  72 struct __asan_global {
  73         const void *beg;                /* address of the global variable */
  74         size_t size;                    /* size of the global variable */
  75         size_t size_with_redzone;       /* size with the redzone */
  76         const void *name;               /* name of the variable */
  77         const void *module_name;        /* name of the module where the var is declared */
  78         unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
  79         struct __asan_global_source_location *location;
  80 #if ASAN_ABI_VERSION >= 7
  81         uintptr_t odr_indicator;        /* the address of the ODR indicator symbol */
  82 #endif
  83 };
  84
  85 FEATURE(kasan, "Kernel address sanitizer");
  86
  87 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  88     "KASAN options");
  89
  90 static int panic_on_violation = 1;
  91 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN,
  92     &panic_on_violation, 0,
  93     "Panic if an invalid access is detected");
  94
  95 #define kasan_enabled (!kasan_disabled)
  96 static bool kasan_disabled __read_mostly = true;
  97 SYSCTL_BOOL(_debug_kasan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
  98     &kasan_disabled, 0, "KASAN is disabled");
  99
 100 /* -------------------------------------------------------------------------- */
 101
 102 void
 103 kasan_shadow_map(vm_offset_t addr, size_t size)
 104 {
 105         size_t sz, npages, i;
 106         vm_offset_t sva, eva;
 107
 108         KASSERT(addr % KASAN_SHADOW_SCALE == 0,
 109             ("%s: invalid address %#lx", __func__, addr));
 110
 111         sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE;
 112
 113         sva = kasan_md_addr_to_shad(addr);
 114         eva = kasan_md_addr_to_shad(addr) + sz;
 115
 116         sva = rounddown(sva, PAGE_SIZE);
 117         eva = roundup(eva, PAGE_SIZE);
 118
 119         npages = (eva - sva) / PAGE_SIZE;
 120
 121         KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS,
 122             ("%s: invalid address range %#lx-%#lx", __func__, sva, eva));
 123
 124         for (i = 0; i < npages; i++)
 125                 pmap_san_enter(sva + ptoa(i));
 126 }
 127
 128 void
 129 kasan_init(void)
 130 {
 131         int disabled;
 132
 133         disabled = 0;
 134         TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled);
 135         if (disabled)
 136                 return;
 137
 138         /* MD initialization. */
 139         kasan_md_init();
 140
 141         /* Now officially enabled. */
 142         kasan_disabled = false;
 143 }
 144
 145 void
 146 kasan_init_early(vm_offset_t stack, size_t size)
 147 {
 148         kasan_md_init_early(stack, size);
 149 }
 150
 151 static inline const char *
 152 kasan_code_name(uint8_t code)
 153 {
 154         switch (code) {
 155         case KASAN_GENERIC_REDZONE:
 156                 return "GenericRedZone";
 157         case KASAN_MALLOC_REDZONE:
 158                 return "MallocRedZone";
 159         case KASAN_KMEM_REDZONE:
 160                 return "KmemRedZone";
 161         case KASAN_UMA_FREED:
 162                 return "UMAUseAfterFree";
 163         case KASAN_KSTACK_FREED:
 164                 return "KernelStack";
 165         case KASAN_EXEC_ARGS_FREED:
 166                 return "ExecKVA";
 167         case 1 ... 7:
 168                 return "RedZonePartial";
 169         case KASAN_STACK_LEFT:
 170                 return "StackLeft";
 171         case KASAN_STACK_MID:
 172                 return "StackMiddle";
 173         case KASAN_STACK_RIGHT:
 174                 return "StackRight";
 175         case KASAN_USE_AFTER_RET:
 176                 return "UseAfterRet";
 177         case KASAN_USE_AFTER_SCOPE:
 178                 return "UseAfterScope";
 179         default:
 180                 return "Unknown";
 181         }
 182 }
 183
 184 #define REPORT(f, ...) do {                             \
 185         if (panic_on_violation) {                       \
 186                 kasan_disabled = true;                  \
 187                 panic(f, __VA_ARGS__);                  \
 188         } else {                                        \
 189                 struct stack st;                        \
 190                                                         \
 191                 stack_save(&st);                        \
 192                 printf(f "\n", __VA_ARGS__);            \
 193                 stack_print_ddb(&st);                   \
 194         }                                               \
 195 } while (0)
 196
 197 static void
 198 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
 199     uint8_t code)
 200 {
 201         REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)",
 202             size, (write ? "write" : "read"), addr, kasan_code_name(code),
 203             code);
 204 }
 205
 206 static __always_inline void
 207 kasan_shadow_1byte_markvalid(unsigned long addr)
 208 {
 209         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 210         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 211
 212         *byte = last;
 213 }
 214
 215 static __always_inline void
 216 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
 217 {
 218         size_t i;
 219
 220         for (i = 0; i < size; i++) {
 221                 kasan_shadow_1byte_markvalid((unsigned long)addr + i);
 222         }
 223 }
 224
 225 static __always_inline void
 226 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
 227 {
 228         void *shad;
 229
 230         if (__predict_false(size == 0))
 231                 return;
 232         if (__predict_false(kasan_md_unsupported((vm_offset_t)addr)))
 233                 return;
 234
 235         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 236             ("%s: invalid address %p", __func__, addr));
 237         KASSERT(size % KASAN_SHADOW_SCALE == 0,
 238             ("%s: invalid size %zu", __func__, size));
 239
 240         shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr);
 241         size = size >> KASAN_SHADOW_SCALE_SHIFT;
 242
 243         __builtin_memset(shad, code, size);
 244 }
 245
 246 /*
 247  * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
 248  * and the rest as invalid. There are generally two use cases:
 249  *
 250  *  o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
 251  *    the redzone at the end of the buffer as invalid. If the entire is to be
 252  *    marked invalid, origsize will be 0.
 253  *
 254  *  o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
 255  */
 256 void
 257 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
 258 {
 259         size_t i, n, redz;
 260         int8_t *shad;
 261
 262         if (__predict_false(!kasan_enabled))
 263                 return;
 264
 265         if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS &&
 266             (vm_offset_t)addr < DMAP_MAX_ADDRESS)
 267                 return;
 268
 269         KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS &&
 270             (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS,
 271             ("%s: invalid address %p", __func__, addr));
 272         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 273             ("%s: invalid address %p", __func__, addr));
 274         redz = redzsize - roundup(size, KASAN_SHADOW_SCALE);
 275         KASSERT(redz % KASAN_SHADOW_SCALE == 0,
 276             ("%s: invalid size %zu", __func__, redz));
 277         shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr);
 278
 279         /* Chunks of 8 bytes, valid. */
 280         n = size / KASAN_SHADOW_SCALE;
 281         for (i = 0; i < n; i++) {
 282                 *shad++ = 0;
 283         }
 284
 285         /* Possibly one chunk, mid. */
 286         if ((size & KASAN_SHADOW_MASK) != 0) {
 287                 *shad++ = (size & KASAN_SHADOW_MASK);
 288         }
 289
 290         /* Chunks of 8 bytes, invalid. */
 291         n = redz / KASAN_SHADOW_SCALE;
 292         for (i = 0; i < n; i++) {
 293                 *shad++ = code;
 294         }
 295 }
 296
 297 /* -------------------------------------------------------------------------- */
 298
 299 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size)                 \
 300         (addr >> KASAN_SHADOW_SCALE_SHIFT) !=                   \
 301             ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
 302
 303 static __always_inline bool
 304 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
 305 {
 306         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 307         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 308
 309         if (__predict_true(*byte == 0 || last <= *byte)) {
 310                 return (true);
 311         }
 312         *code = *byte;
 313         return (false);
 314 }
 315
 316 static __always_inline bool
 317 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
 318 {
 319         int8_t *byte, last;
 320
 321         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
 322                 return (kasan_shadow_1byte_isvalid(addr, code) &&
 323                     kasan_shadow_1byte_isvalid(addr+1, code));
 324         }
 325
 326         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 327         last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
 328
 329         if (__predict_true(*byte == 0 || last <= *byte)) {
 330                 return (true);
 331         }
 332         *code = *byte;
 333         return (false);
 334 }
 335
 336 static __always_inline bool
 337 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
 338 {
 339         int8_t *byte, last;
 340
 341         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
 342                 return (kasan_shadow_2byte_isvalid(addr, code) &&
 343                     kasan_shadow_2byte_isvalid(addr+2, code));
 344         }
 345
 346         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 347         last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
 348
 349         if (__predict_true(*byte == 0 || last <= *byte)) {
 350                 return (true);
 351         }
 352         *code = *byte;
 353         return (false);
 354 }
 355
 356 static __always_inline bool
 357 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
 358 {
 359         int8_t *byte, last;
 360
 361         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
 362                 return (kasan_shadow_4byte_isvalid(addr, code) &&
 363                     kasan_shadow_4byte_isvalid(addr+4, code));
 364         }
 365
 366         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 367         last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
 368
 369         if (__predict_true(*byte == 0 || last <= *byte)) {
 370                 return (true);
 371         }
 372         *code = *byte;
 373         return (false);
 374 }
 375
 376 static __always_inline bool
 377 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
 378 {
 379         size_t i;
 380
 381         for (i = 0; i < size; i++) {
 382                 if (!kasan_shadow_1byte_isvalid(addr+i, code))
 383                         return (false);
 384         }
 385
 386         return (true);
 387 }
 388
 389 static __always_inline void
 390 kasan_shadow_check(unsigned long addr, size_t size, bool write,
 391     unsigned long retaddr)
 392 {
 393         uint8_t code;
 394         bool valid;
 395
 396         if (__predict_false(!kasan_enabled))
 397                 return;
 398         if (__predict_false(size == 0))
 399                 return;
 400         if (__predict_false(kasan_md_unsupported(addr)))
 401                 return;
 402         if (KERNEL_PANICKED())
 403                 return;
 404
 405         if (__builtin_constant_p(size)) {
 406                 switch (size) {
 407                 case 1:
 408                         valid = kasan_shadow_1byte_isvalid(addr, &code);
 409                         break;
 410                 case 2:
 411                         valid = kasan_shadow_2byte_isvalid(addr, &code);
 412                         break;
 413                 case 4:
 414                         valid = kasan_shadow_4byte_isvalid(addr, &code);
 415                         break;
 416                 case 8:
 417                         valid = kasan_shadow_8byte_isvalid(addr, &code);
 418                         break;
 419                 default:
 420                         valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 421                         break;
 422                 }
 423         } else {
 424                 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 425         }
 426
 427         if (__predict_false(!valid)) {
 428                 kasan_report(addr, size, write, retaddr, code);
 429         }
 430 }
 431
 432 /* -------------------------------------------------------------------------- */
 433
 434 void *
 435 kasan_memcpy(void *dst, const void *src, size_t len)
 436 {
 437         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 438         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 439         return (__builtin_memcpy(dst, src, len));
 440 }
 441
 442 int
 443 kasan_memcmp(const void *b1, const void *b2, size_t len)
 444 {
 445         kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
 446         kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
 447         return (__builtin_memcmp(b1, b2, len));
 448 }
 449
 450 void *
 451 kasan_memset(void *b, int c, size_t len)
 452 {
 453         kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
 454         return (__builtin_memset(b, c, len));
 455 }
 456
 457 void *
 458 kasan_memmove(void *dst, const void *src, size_t len)
 459 {
 460         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 461         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 462         return (__builtin_memmove(dst, src, len));
 463 }
 464
 465 size_t
 466 kasan_strlen(const char *str)
 467 {
 468         const char *s;
 469
 470         s = str;
 471         while (1) {
 472                 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
 473                 if (*s == '\0')
 474                         break;
 475                 s++;
 476         }
 477
 478         return (s - str);
 479 }
 480
 481 char *
 482 kasan_strcpy(char *dst, const char *src)
 483 {
 484         char *save = dst;
 485
 486         while (1) {
 487                 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
 488                 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
 489                 *dst = *src;
 490                 if (*src == '\0')
 491                         break;
 492                 src++, dst++;
 493         }
 494
 495         return save;
 496 }
 497
 498 int
 499 kasan_strcmp(const char *s1, const char *s2)
 500 {
 501         while (1) {
 502                 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
 503                 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
 504                 if (*s1 != *s2)
 505                         break;
 506                 if (*s1 == '\0')
 507                         return 0;
 508                 s1++, s2++;
 509         }
 510
 511         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 512 }
 513
 514 int
 515 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
 516 {
 517         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 518         return (copyin(uaddr, kaddr, len));
 519 }
 520
 521 int
 522 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 523 {
 524         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 525         return (copyinstr(uaddr, kaddr, len, done));
 526 }
 527
 528 int
 529 kasan_copyout(const void *kaddr, void *uaddr, size_t len)
 530 {
 531         kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
 532         return (copyout(kaddr, uaddr, len));
 533 }
 534
 535 /* -------------------------------------------------------------------------- */
 536
 537 int
 538 kasan_fubyte(volatile const void *base)
 539 {
 540         return (fubyte(base));
 541 }
 542
 543 int
 544 kasan_fuword16(volatile const void *base)
 545 {
 546         return (fuword16(base));
 547 }
 548
 549 int
 550 kasan_fueword(volatile const void *base, long *val)
 551 {
 552         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 553         return (fueword(base, val));
 554 }
 555
 556 int
 557 kasan_fueword32(volatile const void *base, int32_t *val)
 558 {
 559         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 560         return (fueword32(base, val));
 561 }
 562
 563 int
 564 kasan_fueword64(volatile const void *base, int64_t *val)
 565 {
 566         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 567         return (fueword64(base, val));
 568 }
 569
 570 int
 571 kasan_subyte(volatile void *base, int byte)
 572 {
 573         return (subyte(base, byte));
 574 }
 575
 576 int
 577 kasan_suword(volatile void *base, long word)
 578 {
 579         return (suword(base, word));
 580 }
 581
 582 int
 583 kasan_suword16(volatile void *base, int word)
 584 {
 585         return (suword16(base, word));
 586 }
 587
 588 int
 589 kasan_suword32(volatile void *base, int32_t word)
 590 {
 591         return (suword32(base, word));
 592 }
 593
 594 int
 595 kasan_suword64(volatile void *base, int64_t word)
 596 {
 597         return (suword64(base, word));
 598 }
 599
 600 int
 601 kasan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
 602     uint32_t newval)
 603 {
 604         kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
 605             __RET_ADDR);
 606         return (casueword32(base, oldval, oldvalp, newval));
 607 }
 608
 609 int
 610 kasan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
 611     u_long newval)
 612 {
 613         kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
 614             __RET_ADDR);
 615         return (casueword(base, oldval, oldvalp, newval));
 616 }
 617
 618 /* -------------------------------------------------------------------------- */
 619
 620 #include <machine/atomic.h>
 621 #include <sys/atomic_san.h>
 622
 623 #define _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 624         void kasan_atomic_add_##name(volatile type *ptr, type val)      \
 625         {                                                               \
 626                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 627                     __RET_ADDR);                                        \
 628                 atomic_add_##name(ptr, val);                            \
 629         }
 630
 631 #define ASAN_ATOMIC_FUNC_ADD(name, type)                                \
 632         _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 633         _ASAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
 634         _ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
 635
 636 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 637         void kasan_atomic_subtract_##name(volatile type *ptr, type val) \
 638         {                                                               \
 639                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 640                     __RET_ADDR);                                        \
 641                 atomic_subtract_##name(ptr, val);                       \
 642         }
 643
 644 #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
 645         _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 646         _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
 647         _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
 648
 649 #define _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 650         void kasan_atomic_set_##name(volatile type *ptr, type val)      \
 651         {                                                               \
 652                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 653                     __RET_ADDR);                                        \
 654                 atomic_set_##name(ptr, val);                            \
 655         }
 656
 657 #define ASAN_ATOMIC_FUNC_SET(name, type)                                \
 658         _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 659         _ASAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
 660         _ASAN_ATOMIC_FUNC_SET(rel_##name, type)
 661
 662 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 663         void kasan_atomic_clear_##name(volatile type *ptr, type val)    \
 664         {                                                               \
 665                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 666                     __RET_ADDR);                                        \
 667                 atomic_clear_##name(ptr, val);                          \
 668         }
 669
 670 #define ASAN_ATOMIC_FUNC_CLEAR(name, type)                              \
 671         _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 672         _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
 673         _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
 674
 675 #define ASAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
 676         type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \
 677         {                                                               \
 678                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 679                     __RET_ADDR);                                        \
 680                 return (atomic_fetchadd_##name(ptr, val));              \
 681         }
 682
 683 #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
 684         type kasan_atomic_readandclear_##name(volatile type *ptr)       \
 685         {                                                               \
 686                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 687                     __RET_ADDR);                                        \
 688                 return (atomic_readandclear_##name(ptr));               \
 689         }
 690
 691 #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
 692         int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
 693         {                                                               \
 694                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 695                     __RET_ADDR);                                        \
 696                 return (atomic_testandclear_##name(ptr, v));            \
 697         }
 698
 699 #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
 700         int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
 701         {                                                               \
 702                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 703                     __RET_ADDR);                                        \
 704                 return (atomic_testandset_##name(ptr, v));              \
 705         }
 706
 707 #define ASAN_ATOMIC_FUNC_SWAP(name, type)                               \
 708         type kasan_atomic_swap_##name(volatile type *ptr, type val)     \
 709         {                                                               \
 710                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 711                     __RET_ADDR);                                        \
 712                 return (atomic_swap_##name(ptr, val));                  \
 713         }
 714
 715 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 716         int kasan_atomic_cmpset_##name(volatile type *ptr, type oval,   \
 717             type nval)                                                  \
 718         {                                                               \
 719                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 720                     __RET_ADDR);                                        \
 721                 return (atomic_cmpset_##name(ptr, oval, nval));         \
 722         }
 723
 724 #define ASAN_ATOMIC_FUNC_CMPSET(name, type)                             \
 725         _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 726         _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
 727         _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
 728
 729 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 730         int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
 731             type nval)                                                  \
 732         {                                                               \
 733                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 734                     __RET_ADDR);                                        \
 735                 return (atomic_fcmpset_##name(ptr, oval, nval));        \
 736         }
 737
 738 #define ASAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
 739         _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 740         _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
 741         _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
 742
 743 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
 744         void kasan_atomic_thread_fence_##name(void)                     \
 745         {                                                               \
 746                 atomic_thread_fence_##name();                           \
 747         }
 748
 749 #define _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 750         type kasan_atomic_load_##name(volatile type *ptr)               \
 751         {                                                               \
 752                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 753                     __RET_ADDR);                                        \
 754                 return (atomic_load_##name(ptr));                       \
 755         }
 756
 757 #define ASAN_ATOMIC_FUNC_LOAD(name, type)                               \
 758         _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 759         _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
 760
 761 #define _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 762         void kasan_atomic_store_##name(volatile type *ptr, type val)    \
 763         {                                                               \
 764                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 765                     __RET_ADDR);                                        \
 766                 atomic_store_##name(ptr, val);                          \
 767         }
 768
 769 #define ASAN_ATOMIC_FUNC_STORE(name, type)                              \
 770         _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 771         _ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
 772
 773 ASAN_ATOMIC_FUNC_ADD(8, uint8_t);
 774 ASAN_ATOMIC_FUNC_ADD(16, uint16_t);
 775 ASAN_ATOMIC_FUNC_ADD(32, uint32_t);
 776 ASAN_ATOMIC_FUNC_ADD(64, uint64_t);
 777 ASAN_ATOMIC_FUNC_ADD(int, u_int);
 778 ASAN_ATOMIC_FUNC_ADD(long, u_long);
 779 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
 780
 781 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
 782 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
 783 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
 784 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
 785 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
 786 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
 787 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
 788
 789 ASAN_ATOMIC_FUNC_SET(8, uint8_t);
 790 ASAN_ATOMIC_FUNC_SET(16, uint16_t);
 791 ASAN_ATOMIC_FUNC_SET(32, uint32_t);
 792 ASAN_ATOMIC_FUNC_SET(64, uint64_t);
 793 ASAN_ATOMIC_FUNC_SET(int, u_int);
 794 ASAN_ATOMIC_FUNC_SET(long, u_long);
 795 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
 796
 797 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
 798 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
 799 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
 800 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
 801 ASAN_ATOMIC_FUNC_CLEAR(int, u_int);
 802 ASAN_ATOMIC_FUNC_CLEAR(long, u_long);
 803 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
 804
 805 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
 806 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
 807 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int);
 808 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long);
 809
 810 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
 811 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
 812 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
 813 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
 814 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
 815
 816 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
 817 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
 818 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
 819 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
 820
 821 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
 822 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
 823 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
 824 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
 825
 826 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t);
 827 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t);
 828 ASAN_ATOMIC_FUNC_SWAP(int, u_int);
 829 ASAN_ATOMIC_FUNC_SWAP(long, u_long);
 830 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
 831
 832 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
 833 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
 834 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
 835 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
 836 ASAN_ATOMIC_FUNC_CMPSET(int, u_int);
 837 ASAN_ATOMIC_FUNC_CMPSET(long, u_long);
 838 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
 839
 840 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
 841 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
 842 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
 843 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
 844 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int);
 845 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long);
 846 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
 847
 848 _ASAN_ATOMIC_FUNC_LOAD(bool, bool);
 849 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t);
 850 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t);
 851 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t);
 852 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t);
 853 ASAN_ATOMIC_FUNC_LOAD(char, u_char);
 854 ASAN_ATOMIC_FUNC_LOAD(short, u_short);
 855 ASAN_ATOMIC_FUNC_LOAD(int, u_int);
 856 ASAN_ATOMIC_FUNC_LOAD(long, u_long);
 857 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
 858
 859 _ASAN_ATOMIC_FUNC_STORE(bool, bool);
 860 ASAN_ATOMIC_FUNC_STORE(8, uint8_t);
 861 ASAN_ATOMIC_FUNC_STORE(16, uint16_t);
 862 ASAN_ATOMIC_FUNC_STORE(32, uint32_t);
 863 ASAN_ATOMIC_FUNC_STORE(64, uint64_t);
 864 ASAN_ATOMIC_FUNC_STORE(char, u_char);
 865 ASAN_ATOMIC_FUNC_STORE(short, u_short);
 866 ASAN_ATOMIC_FUNC_STORE(int, u_int);
 867 ASAN_ATOMIC_FUNC_STORE(long, u_long);
 868 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
 869
 870 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq);
 871 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel);
 872 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
 873 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
 874
 875 void
 876 kasan_atomic_interrupt_fence(void)
 877 {
 878 }
 879
 880 /* -------------------------------------------------------------------------- */
 881
 882 #include <sys/bus.h>
 883 #include <machine/bus.h>
 884 #include <sys/bus_san.h>
 885
 886 int
 887 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
 888     int flags, bus_space_handle_t *handlep)
 889 {
 890         return (bus_space_map(tag, hnd, size, flags, handlep));
 891 }
 892
 893 void
 894 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
 895     bus_size_t size)
 896 {
 897         bus_space_unmap(tag, hnd, size);
 898 }
 899
 900 int
 901 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
 902     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
 903 {
 904         return (bus_space_subregion(tag, hnd, offset, size, handlep));
 905 }
 906
 907 void
 908 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
 909     bus_size_t size)
 910 {
 911         bus_space_free(tag, hnd, size);
 912 }
 913
 914 void
 915 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
 916     bus_size_t offset, bus_size_t size, int flags)
 917 {
 918         bus_space_barrier(tag, hnd, offset, size, flags);
 919 }
 920
 921 #define ASAN_BUS_READ_FUNC(func, width, type)                           \
 922         type kasan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
 923             bus_space_handle_t hnd, bus_size_t offset)                  \
 924         {                                                               \
 925                 return (bus_space_read##func##_##width(tag, hnd,        \
 926                     offset));                                           \
 927         }                                                               \
 928
 929 #define ASAN_BUS_READ_PTR_FUNC(func, width, type)                       \
 930         void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
 931             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
 932             bus_size_t count)                                           \
 933         {                                                               \
 934                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 935                     false, __RET_ADDR);                                 \
 936                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
 937                     count);                                             \
 938         }
 939
 940 ASAN_BUS_READ_FUNC(, 1, uint8_t)
 941 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t)
 942 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
 943 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
 944 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
 945 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
 946
 947 ASAN_BUS_READ_FUNC(, 2, uint16_t)
 948 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t)
 949 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
 950 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
 951 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
 952 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
 953
 954 ASAN_BUS_READ_FUNC(, 4, uint32_t)
 955 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t)
 956 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
 957 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
 958 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
 959 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
 960
 961 ASAN_BUS_READ_FUNC(, 8, uint64_t)
 962 #if defined(__aarch64__)
 963 ASAN_BUS_READ_FUNC(_stream, 8, uint64_t)
 964 ASAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
 965 ASAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
 966 ASAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
 967 ASAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
 968 #endif
 969
 970 #define ASAN_BUS_WRITE_FUNC(func, width, type)                          \
 971         void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \
 972             bus_space_handle_t hnd, bus_size_t offset, type value)      \
 973         {                                                               \
 974                 bus_space_write##func##_##width(tag, hnd, offset, value);\
 975         }                                                               \
 976
 977 #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
 978         void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
 979             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
 980             bus_size_t count)                                           \
 981         {                                                               \
 982                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 983                     true, __RET_ADDR);                                  \
 984                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
 985                     count);                                             \
 986         }
 987
 988 ASAN_BUS_WRITE_FUNC(, 1, uint8_t)
 989 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
 990 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
 991 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
 992 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
 993 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
 994
 995 ASAN_BUS_WRITE_FUNC(, 2, uint16_t)
 996 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
 997 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
 998 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
 999 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1000 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1001
1002 ASAN_BUS_WRITE_FUNC(, 4, uint32_t)
1003 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1004 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1005 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1006 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1007 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1008
1009 ASAN_BUS_WRITE_FUNC(, 8, uint64_t)
1010
1011 #define ASAN_BUS_SET_FUNC(func, width, type)                            \
1012         void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
1013             bus_space_handle_t hnd, bus_size_t offset, type value,      \
1014             bus_size_t count)                                           \
1015         {                                                               \
1016                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1017                     count);                                             \
1018         }
1019
1020 ASAN_BUS_SET_FUNC(multi, 1, uint8_t)
1021 ASAN_BUS_SET_FUNC(region, 1, uint8_t)
1022 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1023 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1024
1025 ASAN_BUS_SET_FUNC(multi, 2, uint16_t)
1026 ASAN_BUS_SET_FUNC(region, 2, uint16_t)
1027 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1028 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1029
1030 ASAN_BUS_SET_FUNC(multi, 4, uint32_t)
1031 ASAN_BUS_SET_FUNC(region, 4, uint32_t)
1032 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1033 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1034
1035 #define ASAN_BUS_PEEK_FUNC(width, type)                                 \
1036         int kasan_bus_space_peek_##width(bus_space_tag_t tag,           \
1037             bus_space_handle_t hnd, bus_size_t offset, type *valuep)    \
1038         {                                                               \
1039                 return (bus_space_peek_##width(tag, hnd, offset,        \
1040                     valuep));                                           \
1041         }
1042
1043 ASAN_BUS_PEEK_FUNC(1, uint8_t)
1044 ASAN_BUS_PEEK_FUNC(2, uint16_t)
1045 ASAN_BUS_PEEK_FUNC(4, uint32_t)
1046 ASAN_BUS_PEEK_FUNC(8, uint64_t)
1047
1048 #define ASAN_BUS_POKE_FUNC(width, type)                                 \
1049         int kasan_bus_space_poke_##width(bus_space_tag_t tag,           \
1050             bus_space_handle_t hnd, bus_size_t offset, type value)      \
1051         {                                                               \
1052                 return (bus_space_poke_##width(tag, hnd, offset,        \
1053                     value));                                            \
1054         }
1055
1056 ASAN_BUS_POKE_FUNC(1, uint8_t)
1057 ASAN_BUS_POKE_FUNC(2, uint16_t)
1058 ASAN_BUS_POKE_FUNC(4, uint32_t)
1059 ASAN_BUS_POKE_FUNC(8, uint64_t)
1060
1061 /* -------------------------------------------------------------------------- */
1062
1063 void __asan_register_globals(struct __asan_global *, size_t);
1064 void __asan_unregister_globals(struct __asan_global *, size_t);
1065
1066 void
1067 __asan_register_globals(struct __asan_global *globals, size_t n)
1068 {
1069         size_t i;
1070
1071         for (i = 0; i < n; i++) {
1072                 kasan_mark(globals[i].beg, globals[i].size,
1073                     globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
1074         }
1075 }
1076
1077 void
1078 __asan_unregister_globals(struct __asan_global *globals, size_t n)
1079 {
1080         size_t i;
1081
1082         for (i = 0; i < n; i++) {
1083                 kasan_mark(globals[i].beg, globals[i].size_with_redzone,
1084                     globals[i].size_with_redzone, 0);
1085         }
1086 }
1087
1088 #define ASAN_LOAD_STORE(size)                                   \
1089         void __asan_load##size(unsigned long);                  \
1090         void __asan_load##size(unsigned long addr)              \
1091         {                                                       \
1092                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1093         }                                                       \
1094         void __asan_load##size##_noabort(unsigned long);        \
1095         void __asan_load##size##_noabort(unsigned long addr)    \
1096         {                                                       \
1097                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1098         }                                                       \
1099         void __asan_store##size(unsigned long);                 \
1100         void __asan_store##size(unsigned long addr)             \
1101         {                                                       \
1102                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1103         }                                                       \
1104         void __asan_store##size##_noabort(unsigned long);       \
1105         void __asan_store##size##_noabort(unsigned long addr)   \
1106         {                                                       \
1107                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1108         }
1109
1110 ASAN_LOAD_STORE(1);
1111 ASAN_LOAD_STORE(2);
1112 ASAN_LOAD_STORE(4);
1113 ASAN_LOAD_STORE(8);
1114 ASAN_LOAD_STORE(16);
1115
1116 void __asan_loadN(unsigned long, size_t);
1117 void __asan_loadN_noabort(unsigned long, size_t);
1118 void __asan_storeN(unsigned long, size_t);
1119 void __asan_storeN_noabort(unsigned long, size_t);
1120 void __asan_handle_no_return(void);
1121
1122 void
1123 __asan_loadN(unsigned long addr, size_t size)
1124 {
1125         kasan_shadow_check(addr, size, false, __RET_ADDR);
1126 }
1127
1128 void
1129 __asan_loadN_noabort(unsigned long addr, size_t size)
1130 {
1131         kasan_shadow_check(addr, size, false, __RET_ADDR);
1132 }
1133
1134 void
1135 __asan_storeN(unsigned long addr, size_t size)
1136 {
1137         kasan_shadow_check(addr, size, true, __RET_ADDR);
1138 }
1139
1140 void
1141 __asan_storeN_noabort(unsigned long addr, size_t size)
1142 {
1143         kasan_shadow_check(addr, size, true, __RET_ADDR);
1144 }
1145
1146 void
1147 __asan_handle_no_return(void)
1148 {
1149         /* nothing */
1150 }
1151
1152 #define ASAN_SET_SHADOW(byte) \
1153         void __asan_set_shadow_##byte(void *, size_t);                  \
1154         void __asan_set_shadow_##byte(void *addr, size_t size)          \
1155         {                                                               \
1156                 __builtin_memset((void *)addr, 0x##byte, size);         \
1157         }
1158
1159 ASAN_SET_SHADOW(00);
1160 ASAN_SET_SHADOW(f1);
1161 ASAN_SET_SHADOW(f2);
1162 ASAN_SET_SHADOW(f3);
1163 ASAN_SET_SHADOW(f5);
1164 ASAN_SET_SHADOW(f8);
1165
1166 void __asan_poison_stack_memory(const void *, size_t);
1167 void __asan_unpoison_stack_memory(const void *, size_t);
1168
1169 void
1170 __asan_poison_stack_memory(const void *addr, size_t size)
1171 {
1172         size = roundup(size, KASAN_SHADOW_SCALE);
1173         kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1174 }
1175
1176 void
1177 __asan_unpoison_stack_memory(const void *addr, size_t size)
1178 {
1179         kasan_shadow_Nbyte_markvalid(addr, size);
1180 }
1181
1182 void __asan_alloca_poison(const void *, size_t);
1183 void __asan_allocas_unpoison(const void *, const void *);
1184
1185 void
1186 __asan_alloca_poison(const void *addr, size_t size)
1187 {
1188         const void *l, *r;
1189
1190         KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0,
1191             ("%s: invalid address %p", __func__, addr));
1192
1193         l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1194         r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1195
1196         kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1197         kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1198             KASAN_STACK_MID);
1199         kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1200 }
1201
1202 void
1203 __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1204 {
1205         size_t size;
1206
1207         if (__predict_false(!stkbegin))
1208                 return;
1209         if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1210                 return;
1211         size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1212
1213         kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1214 }
1215
1216 void __asan_poison_memory_region(const void *addr, size_t size);
1217 void __asan_unpoison_memory_region(const void *addr, size_t size);
1218
1219 void
1220 __asan_poison_memory_region(const void *addr, size_t size)
1221 {
1222 }
1223
1224 void
1225 __asan_unpoison_memory_region(const void *addr, size_t size)
1226 {
1227 }