sys/kern/subr_asan.c

   1 /*      $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $      */
   2
   3 /*
   4  * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
   5  * All rights reserved.
   6  *
   7  * This code is part of the KASAN subsystem of the NetBSD kernel.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  23  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  25  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  26  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 #define SAN_RUNTIME
  32
  33 #include <sys/cdefs.h>
  34 #if 0
  35 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $");
  36 #endif
  37
  38 #include <sys/param.h>
  39 #include <sys/systm.h>
  40 #include <sys/asan.h>
  41 #include <sys/kernel.h>
  42 #include <sys/proc.h>
  43 #include <sys/stack.h>
  44 #include <sys/sysctl.h>
  45
  46 #include <machine/asan.h>
  47 #include <machine/bus.h>
  48
  49 /* ASAN constants. Part of the compiler ABI. */
  50 #define KASAN_SHADOW_MASK               (KASAN_SHADOW_SCALE - 1)
  51 #define KASAN_ALLOCA_SCALE_SIZE         32
  52
  53 /* ASAN ABI version. */
  54 #if defined(__clang__) && (__clang_major__ - 0 >= 6)
  55 #define ASAN_ABI_VERSION        8
  56 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
  57 #define ASAN_ABI_VERSION        8
  58 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
  59 #define ASAN_ABI_VERSION        6
  60 #else
  61 #error "Unsupported compiler version"
  62 #endif
  63
  64 #define __RET_ADDR      (unsigned long)__builtin_return_address(0)
  65
  66 /* Global variable descriptor. Part of the compiler ABI.  */
  67 struct __asan_global_source_location {
  68         const char *filename;
  69         int line_no;
  70         int column_no;
  71 };
  72
  73 struct __asan_global {
  74         const void *beg;                /* address of the global variable */
  75         size_t size;                    /* size of the global variable */
  76         size_t size_with_redzone;       /* size with the redzone */
  77         const void *name;               /* name of the variable */
  78         const void *module_name;        /* name of the module where the var is declared */
  79         unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
  80         struct __asan_global_source_location *location;
  81 #if ASAN_ABI_VERSION >= 7
  82         uintptr_t odr_indicator;        /* the address of the ODR indicator symbol */
  83 #endif
  84 };
  85
  86 FEATURE(kasan, "Kernel address sanitizer");
  87
  88 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  89     "KASAN options");
  90
  91 static int panic_on_violation = 1;
  92 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN,
  93     &panic_on_violation, 0,
  94     "Panic if an invalid access is detected");
  95
  96 #define kasan_enabled (!kasan_disabled)
  97 static bool kasan_disabled __read_mostly = true;
  98 SYSCTL_BOOL(_debug_kasan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
  99     &kasan_disabled, 0, "KASAN is disabled");
 100
 101 /* -------------------------------------------------------------------------- */
 102
 103 void
 104 kasan_shadow_map(vm_offset_t addr, size_t size)
 105 {
 106         size_t sz, npages, i;
 107         vm_offset_t sva, eva;
 108
 109         KASSERT(addr % KASAN_SHADOW_SCALE == 0,
 110             ("%s: invalid address %#lx", __func__, addr));
 111
 112         sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE;
 113
 114         sva = kasan_md_addr_to_shad(addr);
 115         eva = kasan_md_addr_to_shad(addr) + sz;
 116
 117         sva = rounddown(sva, PAGE_SIZE);
 118         eva = roundup(eva, PAGE_SIZE);
 119
 120         npages = (eva - sva) / PAGE_SIZE;
 121
 122         KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS,
 123             ("%s: invalid address range %#lx-%#lx", __func__, sva, eva));
 124
 125         for (i = 0; i < npages; i++)
 126                 pmap_san_enter(sva + ptoa(i));
 127 }
 128
 129 void
 130 kasan_init(void)
 131 {
 132         int disabled;
 133
 134         disabled = 0;
 135         TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled);
 136         if (disabled)
 137                 return;
 138
 139         /* MD initialization. */
 140         kasan_md_init();
 141
 142         /* Now officially enabled. */
 143         kasan_disabled = false;
 144 }
 145
 146 void
 147 kasan_init_early(vm_offset_t stack, size_t size)
 148 {
 149         kasan_md_init_early(stack, size);
 150 }
 151
 152 static inline const char *
 153 kasan_code_name(uint8_t code)
 154 {
 155         switch (code) {
 156         case KASAN_GENERIC_REDZONE:
 157                 return "GenericRedZone";
 158         case KASAN_MALLOC_REDZONE:
 159                 return "MallocRedZone";
 160         case KASAN_KMEM_REDZONE:
 161                 return "KmemRedZone";
 162         case KASAN_UMA_FREED:
 163                 return "UMAUseAfterFree";
 164         case KASAN_KSTACK_FREED:
 165                 return "KernelStack";
 166         case KASAN_EXEC_ARGS_FREED:
 167                 return "ExecKVA";
 168         case 1 ... 7:
 169                 return "RedZonePartial";
 170         case KASAN_STACK_LEFT:
 171                 return "StackLeft";
 172         case KASAN_STACK_MID:
 173                 return "StackMiddle";
 174         case KASAN_STACK_RIGHT:
 175                 return "StackRight";
 176         case KASAN_USE_AFTER_RET:
 177                 return "UseAfterRet";
 178         case KASAN_USE_AFTER_SCOPE:
 179                 return "UseAfterScope";
 180         default:
 181                 return "Unknown";
 182         }
 183 }
 184
 185 #define REPORT(f, ...) do {                             \
 186         if (panic_on_violation) {                       \
 187                 kasan_disabled = true;                  \
 188                 panic(f, __VA_ARGS__);                  \
 189         } else {                                        \
 190                 struct stack st;                        \
 191                                                         \
 192                 stack_save(&st);                        \
 193                 printf(f "\n", __VA_ARGS__);            \
 194                 stack_print_ddb(&st);                   \
 195         }                                               \
 196 } while (0)
 197
 198 static void
 199 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
 200     uint8_t code)
 201 {
 202         REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)",
 203             size, (write ? "write" : "read"), addr, kasan_code_name(code),
 204             code);
 205 }
 206
 207 static __always_inline void
 208 kasan_shadow_1byte_markvalid(unsigned long addr)
 209 {
 210         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 211         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 212
 213         *byte = last;
 214 }
 215
 216 static __always_inline void
 217 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
 218 {
 219         size_t i;
 220
 221         for (i = 0; i < size; i++) {
 222                 kasan_shadow_1byte_markvalid((unsigned long)addr + i);
 223         }
 224 }
 225
 226 static __always_inline void
 227 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
 228 {
 229         void *shad;
 230
 231         if (__predict_false(size == 0))
 232                 return;
 233         if (__predict_false(kasan_md_unsupported((vm_offset_t)addr)))
 234                 return;
 235
 236         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 237             ("%s: invalid address %p", __func__, addr));
 238         KASSERT(size % KASAN_SHADOW_SCALE == 0,
 239             ("%s: invalid size %zu", __func__, size));
 240
 241         shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr);
 242         size = size >> KASAN_SHADOW_SCALE_SHIFT;
 243
 244         __builtin_memset(shad, code, size);
 245 }
 246
 247 /*
 248  * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
 249  * and the rest as invalid. There are generally two use cases:
 250  *
 251  *  o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
 252  *    the redzone at the end of the buffer as invalid. If the entire is to be
 253  *    marked invalid, origsize will be 0.
 254  *
 255  *  o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
 256  */
 257 void
 258 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
 259 {
 260         size_t i, n, redz;
 261         int8_t *shad;
 262
 263         if (__predict_false(!kasan_enabled))
 264                 return;
 265
 266         if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS &&
 267             (vm_offset_t)addr < DMAP_MAX_ADDRESS)
 268                 return;
 269
 270         KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS &&
 271             (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS,
 272             ("%s: invalid address %p", __func__, addr));
 273         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 274             ("%s: invalid address %p", __func__, addr));
 275         redz = redzsize - roundup(size, KASAN_SHADOW_SCALE);
 276         KASSERT(redz % KASAN_SHADOW_SCALE == 0,
 277             ("%s: invalid size %zu", __func__, redz));
 278         shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr);
 279
 280         /* Chunks of 8 bytes, valid. */
 281         n = size / KASAN_SHADOW_SCALE;
 282         for (i = 0; i < n; i++) {
 283                 *shad++ = 0;
 284         }
 285
 286         /* Possibly one chunk, mid. */
 287         if ((size & KASAN_SHADOW_MASK) != 0) {
 288                 *shad++ = (size & KASAN_SHADOW_MASK);
 289         }
 290
 291         /* Chunks of 8 bytes, invalid. */
 292         n = redz / KASAN_SHADOW_SCALE;
 293         for (i = 0; i < n; i++) {
 294                 *shad++ = code;
 295         }
 296 }
 297
 298 void
 299 kasan_thread_alloc(struct thread *td)
 300 {
 301         if (td->td_kstack != 0) {
 302                 kasan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages),
 303                     ptoa(td->td_kstack_pages), 0);
 304         }
 305 }
 306
 307 /* -------------------------------------------------------------------------- */
 308
 309 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size)                 \
 310         (addr >> KASAN_SHADOW_SCALE_SHIFT) !=                   \
 311             ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
 312
 313 static __always_inline bool
 314 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
 315 {
 316         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 317         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 318
 319         if (__predict_true(*byte == 0 || last <= *byte)) {
 320                 return (true);
 321         }
 322         *code = *byte;
 323         return (false);
 324 }
 325
 326 static __always_inline bool
 327 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
 328 {
 329         int8_t *byte, last;
 330
 331         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
 332                 return (kasan_shadow_1byte_isvalid(addr, code) &&
 333                     kasan_shadow_1byte_isvalid(addr+1, code));
 334         }
 335
 336         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 337         last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
 338
 339         if (__predict_true(*byte == 0 || last <= *byte)) {
 340                 return (true);
 341         }
 342         *code = *byte;
 343         return (false);
 344 }
 345
 346 static __always_inline bool
 347 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
 348 {
 349         int8_t *byte, last;
 350
 351         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
 352                 return (kasan_shadow_2byte_isvalid(addr, code) &&
 353                     kasan_shadow_2byte_isvalid(addr+2, code));
 354         }
 355
 356         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 357         last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
 358
 359         if (__predict_true(*byte == 0 || last <= *byte)) {
 360                 return (true);
 361         }
 362         *code = *byte;
 363         return (false);
 364 }
 365
 366 static __always_inline bool
 367 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
 368 {
 369         int8_t *byte, last;
 370
 371         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
 372                 return (kasan_shadow_4byte_isvalid(addr, code) &&
 373                     kasan_shadow_4byte_isvalid(addr+4, code));
 374         }
 375
 376         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 377         last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
 378
 379         if (__predict_true(*byte == 0 || last <= *byte)) {
 380                 return (true);
 381         }
 382         *code = *byte;
 383         return (false);
 384 }
 385
 386 static __always_inline bool
 387 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
 388 {
 389         size_t i;
 390
 391         for (i = 0; i < size; i++) {
 392                 if (!kasan_shadow_1byte_isvalid(addr+i, code))
 393                         return (false);
 394         }
 395
 396         return (true);
 397 }
 398
 399 static __always_inline void
 400 kasan_shadow_check(unsigned long addr, size_t size, bool write,
 401     unsigned long retaddr)
 402 {
 403         uint8_t code;
 404         bool valid;
 405
 406         if (__predict_false(!kasan_enabled))
 407                 return;
 408         if (__predict_false(size == 0))
 409                 return;
 410         if (__predict_false(kasan_md_unsupported(addr)))
 411                 return;
 412         if (KERNEL_PANICKED())
 413                 return;
 414
 415         if (__builtin_constant_p(size)) {
 416                 switch (size) {
 417                 case 1:
 418                         valid = kasan_shadow_1byte_isvalid(addr, &code);
 419                         break;
 420                 case 2:
 421                         valid = kasan_shadow_2byte_isvalid(addr, &code);
 422                         break;
 423                 case 4:
 424                         valid = kasan_shadow_4byte_isvalid(addr, &code);
 425                         break;
 426                 case 8:
 427                         valid = kasan_shadow_8byte_isvalid(addr, &code);
 428                         break;
 429                 default:
 430                         valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 431                         break;
 432                 }
 433         } else {
 434                 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 435         }
 436
 437         if (__predict_false(!valid)) {
 438                 kasan_report(addr, size, write, retaddr, code);
 439         }
 440 }
 441
 442 /* -------------------------------------------------------------------------- */
 443
 444 void *
 445 kasan_memcpy(void *dst, const void *src, size_t len)
 446 {
 447         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 448         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 449         return (__builtin_memcpy(dst, src, len));
 450 }
 451
 452 int
 453 kasan_memcmp(const void *b1, const void *b2, size_t len)
 454 {
 455         kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
 456         kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
 457         return (__builtin_memcmp(b1, b2, len));
 458 }
 459
 460 void *
 461 kasan_memset(void *b, int c, size_t len)
 462 {
 463         kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
 464         return (__builtin_memset(b, c, len));
 465 }
 466
 467 void *
 468 kasan_memmove(void *dst, const void *src, size_t len)
 469 {
 470         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 471         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 472         return (__builtin_memmove(dst, src, len));
 473 }
 474
 475 size_t
 476 kasan_strlen(const char *str)
 477 {
 478         const char *s;
 479
 480         s = str;
 481         while (1) {
 482                 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
 483                 if (*s == '\0')
 484                         break;
 485                 s++;
 486         }
 487
 488         return (s - str);
 489 }
 490
 491 char *
 492 kasan_strcpy(char *dst, const char *src)
 493 {
 494         char *save = dst;
 495
 496         while (1) {
 497                 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
 498                 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
 499                 *dst = *src;
 500                 if (*src == '\0')
 501                         break;
 502                 src++, dst++;
 503         }
 504
 505         return save;
 506 }
 507
 508 int
 509 kasan_strcmp(const char *s1, const char *s2)
 510 {
 511         while (1) {
 512                 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
 513                 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
 514                 if (*s1 != *s2)
 515                         break;
 516                 if (*s1 == '\0')
 517                         return 0;
 518                 s1++, s2++;
 519         }
 520
 521         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 522 }
 523
 524 int
 525 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
 526 {
 527         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 528         return (copyin(uaddr, kaddr, len));
 529 }
 530
 531 int
 532 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 533 {
 534         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 535         return (copyinstr(uaddr, kaddr, len, done));
 536 }
 537
 538 int
 539 kasan_copyout(const void *kaddr, void *uaddr, size_t len)
 540 {
 541         kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
 542         return (copyout(kaddr, uaddr, len));
 543 }
 544
 545 /* -------------------------------------------------------------------------- */
 546
 547 int
 548 kasan_fubyte(volatile const void *base)
 549 {
 550         return (fubyte(base));
 551 }
 552
 553 int
 554 kasan_fuword16(volatile const void *base)
 555 {
 556         return (fuword16(base));
 557 }
 558
 559 int
 560 kasan_fueword(volatile const void *base, long *val)
 561 {
 562         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 563         return (fueword(base, val));
 564 }
 565
 566 int
 567 kasan_fueword32(volatile const void *base, int32_t *val)
 568 {
 569         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 570         return (fueword32(base, val));
 571 }
 572
 573 int
 574 kasan_fueword64(volatile const void *base, int64_t *val)
 575 {
 576         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 577         return (fueword64(base, val));
 578 }
 579
 580 int
 581 kasan_subyte(volatile void *base, int byte)
 582 {
 583         return (subyte(base, byte));
 584 }
 585
 586 int
 587 kasan_suword(volatile void *base, long word)
 588 {
 589         return (suword(base, word));
 590 }
 591
 592 int
 593 kasan_suword16(volatile void *base, int word)
 594 {
 595         return (suword16(base, word));
 596 }
 597
 598 int
 599 kasan_suword32(volatile void *base, int32_t word)
 600 {
 601         return (suword32(base, word));
 602 }
 603
 604 int
 605 kasan_suword64(volatile void *base, int64_t word)
 606 {
 607         return (suword64(base, word));
 608 }
 609
 610 int
 611 kasan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
 612     uint32_t newval)
 613 {
 614         kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
 615             __RET_ADDR);
 616         return (casueword32(base, oldval, oldvalp, newval));
 617 }
 618
 619 int
 620 kasan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
 621     u_long newval)
 622 {
 623         kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
 624             __RET_ADDR);
 625         return (casueword(base, oldval, oldvalp, newval));
 626 }
 627
 628 /* -------------------------------------------------------------------------- */
 629
 630 #include <machine/atomic.h>
 631 #include <sys/atomic_san.h>
 632
 633 #define _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 634         void kasan_atomic_add_##name(volatile type *ptr, type val)      \
 635         {                                                               \
 636                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 637                     __RET_ADDR);                                        \
 638                 atomic_add_##name(ptr, val);                            \
 639         }
 640
 641 #define ASAN_ATOMIC_FUNC_ADD(name, type)                                \
 642         _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 643         _ASAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
 644         _ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
 645
 646 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 647         void kasan_atomic_subtract_##name(volatile type *ptr, type val) \
 648         {                                                               \
 649                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 650                     __RET_ADDR);                                        \
 651                 atomic_subtract_##name(ptr, val);                       \
 652         }
 653
 654 #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
 655         _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 656         _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
 657         _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
 658
 659 #define _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 660         void kasan_atomic_set_##name(volatile type *ptr, type val)      \
 661         {                                                               \
 662                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 663                     __RET_ADDR);                                        \
 664                 atomic_set_##name(ptr, val);                            \
 665         }
 666
 667 #define ASAN_ATOMIC_FUNC_SET(name, type)                                \
 668         _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 669         _ASAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
 670         _ASAN_ATOMIC_FUNC_SET(rel_##name, type)
 671
 672 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 673         void kasan_atomic_clear_##name(volatile type *ptr, type val)    \
 674         {                                                               \
 675                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 676                     __RET_ADDR);                                        \
 677                 atomic_clear_##name(ptr, val);                          \
 678         }
 679
 680 #define ASAN_ATOMIC_FUNC_CLEAR(name, type)                              \
 681         _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 682         _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
 683         _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
 684
 685 #define ASAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
 686         type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \
 687         {                                                               \
 688                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 689                     __RET_ADDR);                                        \
 690                 return (atomic_fetchadd_##name(ptr, val));              \
 691         }
 692
 693 #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
 694         type kasan_atomic_readandclear_##name(volatile type *ptr)       \
 695         {                                                               \
 696                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 697                     __RET_ADDR);                                        \
 698                 return (atomic_readandclear_##name(ptr));               \
 699         }
 700
 701 #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
 702         int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
 703         {                                                               \
 704                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 705                     __RET_ADDR);                                        \
 706                 return (atomic_testandclear_##name(ptr, v));            \
 707         }
 708
 709 #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
 710         int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
 711         {                                                               \
 712                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 713                     __RET_ADDR);                                        \
 714                 return (atomic_testandset_##name(ptr, v));              \
 715         }
 716
 717 #define ASAN_ATOMIC_FUNC_SWAP(name, type)                               \
 718         type kasan_atomic_swap_##name(volatile type *ptr, type val)     \
 719         {                                                               \
 720                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 721                     __RET_ADDR);                                        \
 722                 return (atomic_swap_##name(ptr, val));                  \
 723         }
 724
 725 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 726         int kasan_atomic_cmpset_##name(volatile type *ptr, type oval,   \
 727             type nval)                                                  \
 728         {                                                               \
 729                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 730                     __RET_ADDR);                                        \
 731                 return (atomic_cmpset_##name(ptr, oval, nval));         \
 732         }
 733
 734 #define ASAN_ATOMIC_FUNC_CMPSET(name, type)                             \
 735         _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 736         _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
 737         _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
 738
 739 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 740         int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
 741             type nval)                                                  \
 742         {                                                               \
 743                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 744                     __RET_ADDR);                                        \
 745                 return (atomic_fcmpset_##name(ptr, oval, nval));        \
 746         }
 747
 748 #define ASAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
 749         _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 750         _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
 751         _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
 752
 753 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
 754         void kasan_atomic_thread_fence_##name(void)                     \
 755         {                                                               \
 756                 atomic_thread_fence_##name();                           \
 757         }
 758
 759 #define _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 760         type kasan_atomic_load_##name(volatile type *ptr)               \
 761         {                                                               \
 762                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 763                     __RET_ADDR);                                        \
 764                 return (atomic_load_##name(ptr));                       \
 765         }
 766
 767 #define ASAN_ATOMIC_FUNC_LOAD(name, type)                               \
 768         _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 769         _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
 770
 771 #define _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 772         void kasan_atomic_store_##name(volatile type *ptr, type val)    \
 773         {                                                               \
 774                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 775                     __RET_ADDR);                                        \
 776                 atomic_store_##name(ptr, val);                          \
 777         }
 778
 779 #define ASAN_ATOMIC_FUNC_STORE(name, type)                              \
 780         _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 781         _ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
 782
 783 ASAN_ATOMIC_FUNC_ADD(8, uint8_t);
 784 ASAN_ATOMIC_FUNC_ADD(16, uint16_t);
 785 ASAN_ATOMIC_FUNC_ADD(32, uint32_t);
 786 ASAN_ATOMIC_FUNC_ADD(64, uint64_t);
 787 ASAN_ATOMIC_FUNC_ADD(int, u_int);
 788 ASAN_ATOMIC_FUNC_ADD(long, u_long);
 789 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
 790
 791 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
 792 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
 793 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
 794 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
 795 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
 796 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
 797 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
 798
 799 ASAN_ATOMIC_FUNC_SET(8, uint8_t);
 800 ASAN_ATOMIC_FUNC_SET(16, uint16_t);
 801 ASAN_ATOMIC_FUNC_SET(32, uint32_t);
 802 ASAN_ATOMIC_FUNC_SET(64, uint64_t);
 803 ASAN_ATOMIC_FUNC_SET(int, u_int);
 804 ASAN_ATOMIC_FUNC_SET(long, u_long);
 805 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
 806
 807 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
 808 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
 809 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
 810 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
 811 ASAN_ATOMIC_FUNC_CLEAR(int, u_int);
 812 ASAN_ATOMIC_FUNC_CLEAR(long, u_long);
 813 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
 814
 815 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
 816 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
 817 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int);
 818 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long);
 819
 820 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
 821 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
 822 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
 823 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
 824 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
 825
 826 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
 827 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
 828 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
 829 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
 830
 831 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
 832 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
 833 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
 834 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
 835
 836 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t);
 837 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t);
 838 ASAN_ATOMIC_FUNC_SWAP(int, u_int);
 839 ASAN_ATOMIC_FUNC_SWAP(long, u_long);
 840 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
 841
 842 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
 843 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
 844 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
 845 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
 846 ASAN_ATOMIC_FUNC_CMPSET(int, u_int);
 847 ASAN_ATOMIC_FUNC_CMPSET(long, u_long);
 848 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
 849
 850 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
 851 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
 852 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
 853 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
 854 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int);
 855 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long);
 856 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
 857
 858 _ASAN_ATOMIC_FUNC_LOAD(bool, bool);
 859 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t);
 860 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t);
 861 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t);
 862 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t);
 863 ASAN_ATOMIC_FUNC_LOAD(char, u_char);
 864 ASAN_ATOMIC_FUNC_LOAD(short, u_short);
 865 ASAN_ATOMIC_FUNC_LOAD(int, u_int);
 866 ASAN_ATOMIC_FUNC_LOAD(long, u_long);
 867 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
 868
 869 _ASAN_ATOMIC_FUNC_STORE(bool, bool);
 870 ASAN_ATOMIC_FUNC_STORE(8, uint8_t);
 871 ASAN_ATOMIC_FUNC_STORE(16, uint16_t);
 872 ASAN_ATOMIC_FUNC_STORE(32, uint32_t);
 873 ASAN_ATOMIC_FUNC_STORE(64, uint64_t);
 874 ASAN_ATOMIC_FUNC_STORE(char, u_char);
 875 ASAN_ATOMIC_FUNC_STORE(short, u_short);
 876 ASAN_ATOMIC_FUNC_STORE(int, u_int);
 877 ASAN_ATOMIC_FUNC_STORE(long, u_long);
 878 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
 879
 880 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq);
 881 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel);
 882 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
 883 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
 884
 885 void
 886 kasan_atomic_interrupt_fence(void)
 887 {
 888 }
 889
 890 /* -------------------------------------------------------------------------- */
 891
 892 #include <sys/bus.h>
 893 #include <machine/bus.h>
 894 #include <sys/bus_san.h>
 895
 896 int
 897 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
 898     int flags, bus_space_handle_t *handlep)
 899 {
 900         return (bus_space_map(tag, hnd, size, flags, handlep));
 901 }
 902
 903 void
 904 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
 905     bus_size_t size)
 906 {
 907         bus_space_unmap(tag, hnd, size);
 908 }
 909
 910 int
 911 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
 912     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
 913 {
 914         return (bus_space_subregion(tag, hnd, offset, size, handlep));
 915 }
 916
 917 void
 918 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
 919     bus_size_t size)
 920 {
 921         bus_space_free(tag, hnd, size);
 922 }
 923
 924 void
 925 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
 926     bus_size_t offset, bus_size_t size, int flags)
 927 {
 928         bus_space_barrier(tag, hnd, offset, size, flags);
 929 }
 930
 931 #define ASAN_BUS_READ_FUNC(func, width, type)                           \
 932         type kasan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
 933             bus_space_handle_t hnd, bus_size_t offset)                  \
 934         {                                                               \
 935                 return (bus_space_read##func##_##width(tag, hnd,        \
 936                     offset));                                           \
 937         }                                                               \
 938
 939 #define ASAN_BUS_READ_PTR_FUNC(func, width, type)                       \
 940         void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
 941             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
 942             bus_size_t count)                                           \
 943         {                                                               \
 944                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 945                     false, __RET_ADDR);                                 \
 946                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
 947                     count);                                             \
 948         }
 949
 950 ASAN_BUS_READ_FUNC(, 1, uint8_t)
 951 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t)
 952 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
 953 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
 954 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
 955 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
 956
 957 ASAN_BUS_READ_FUNC(, 2, uint16_t)
 958 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t)
 959 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
 960 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
 961 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
 962 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
 963
 964 ASAN_BUS_READ_FUNC(, 4, uint32_t)
 965 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t)
 966 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
 967 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
 968 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
 969 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
 970
 971 ASAN_BUS_READ_FUNC(, 8, uint64_t)
 972 #if defined(__aarch64__)
 973 ASAN_BUS_READ_FUNC(_stream, 8, uint64_t)
 974 ASAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
 975 ASAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
 976 ASAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
 977 ASAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
 978 #endif
 979
 980 #define ASAN_BUS_WRITE_FUNC(func, width, type)                          \
 981         void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \
 982             bus_space_handle_t hnd, bus_size_t offset, type value)      \
 983         {                                                               \
 984                 bus_space_write##func##_##width(tag, hnd, offset, value);\
 985         }                                                               \
 986
 987 #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
 988         void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
 989             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
 990             bus_size_t count)                                           \
 991         {                                                               \
 992                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 993                     true, __RET_ADDR);                                  \
 994                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
 995                     count);                                             \
 996         }
 997
 998 ASAN_BUS_WRITE_FUNC(, 1, uint8_t)
 999 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
1000 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
1001 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
1002 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
1003 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
1004
1005 ASAN_BUS_WRITE_FUNC(, 2, uint16_t)
1006 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
1007 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
1008 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
1009 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1010 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1011
1012 ASAN_BUS_WRITE_FUNC(, 4, uint32_t)
1013 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1014 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1015 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1016 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1017 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1018
1019 ASAN_BUS_WRITE_FUNC(, 8, uint64_t)
1020
1021 #define ASAN_BUS_SET_FUNC(func, width, type)                            \
1022         void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
1023             bus_space_handle_t hnd, bus_size_t offset, type value,      \
1024             bus_size_t count)                                           \
1025         {                                                               \
1026                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1027                     count);                                             \
1028         }
1029
1030 ASAN_BUS_SET_FUNC(multi, 1, uint8_t)
1031 ASAN_BUS_SET_FUNC(region, 1, uint8_t)
1032 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1033 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1034
1035 ASAN_BUS_SET_FUNC(multi, 2, uint16_t)
1036 ASAN_BUS_SET_FUNC(region, 2, uint16_t)
1037 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1038 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1039
1040 ASAN_BUS_SET_FUNC(multi, 4, uint32_t)
1041 ASAN_BUS_SET_FUNC(region, 4, uint32_t)
1042 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1043 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1044
1045 #define ASAN_BUS_PEEK_FUNC(width, type)                                 \
1046         int kasan_bus_space_peek_##width(bus_space_tag_t tag,           \
1047             bus_space_handle_t hnd, bus_size_t offset, type *valuep)    \
1048         {                                                               \
1049                 return (bus_space_peek_##width(tag, hnd, offset,        \
1050                     valuep));                                           \
1051         }
1052
1053 ASAN_BUS_PEEK_FUNC(1, uint8_t)
1054 ASAN_BUS_PEEK_FUNC(2, uint16_t)
1055 ASAN_BUS_PEEK_FUNC(4, uint32_t)
1056 ASAN_BUS_PEEK_FUNC(8, uint64_t)
1057
1058 #define ASAN_BUS_POKE_FUNC(width, type)                                 \
1059         int kasan_bus_space_poke_##width(bus_space_tag_t tag,           \
1060             bus_space_handle_t hnd, bus_size_t offset, type value)      \
1061         {                                                               \
1062                 return (bus_space_poke_##width(tag, hnd, offset,        \
1063                     value));                                            \
1064         }
1065
1066 ASAN_BUS_POKE_FUNC(1, uint8_t)
1067 ASAN_BUS_POKE_FUNC(2, uint16_t)
1068 ASAN_BUS_POKE_FUNC(4, uint32_t)
1069 ASAN_BUS_POKE_FUNC(8, uint64_t)
1070
1071 /* -------------------------------------------------------------------------- */
1072
1073 void __asan_register_globals(struct __asan_global *, size_t);
1074 void __asan_unregister_globals(struct __asan_global *, size_t);
1075
1076 void
1077 __asan_register_globals(struct __asan_global *globals, size_t n)
1078 {
1079         size_t i;
1080
1081         for (i = 0; i < n; i++) {
1082                 kasan_mark(globals[i].beg, globals[i].size,
1083                     globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
1084         }
1085 }
1086
1087 void
1088 __asan_unregister_globals(struct __asan_global *globals, size_t n)
1089 {
1090         size_t i;
1091
1092         for (i = 0; i < n; i++) {
1093                 kasan_mark(globals[i].beg, globals[i].size_with_redzone,
1094                     globals[i].size_with_redzone, 0);
1095         }
1096 }
1097
1098 #define ASAN_LOAD_STORE(size)                                   \
1099         void __asan_load##size(unsigned long);                  \
1100         void __asan_load##size(unsigned long addr)              \
1101         {                                                       \
1102                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1103         }                                                       \
1104         void __asan_load##size##_noabort(unsigned long);        \
1105         void __asan_load##size##_noabort(unsigned long addr)    \
1106         {                                                       \
1107                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1108         }                                                       \
1109         void __asan_store##size(unsigned long);                 \
1110         void __asan_store##size(unsigned long addr)             \
1111         {                                                       \
1112                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1113         }                                                       \
1114         void __asan_store##size##_noabort(unsigned long);       \
1115         void __asan_store##size##_noabort(unsigned long addr)   \
1116         {                                                       \
1117                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1118         }
1119
1120 ASAN_LOAD_STORE(1);
1121 ASAN_LOAD_STORE(2);
1122 ASAN_LOAD_STORE(4);
1123 ASAN_LOAD_STORE(8);
1124 ASAN_LOAD_STORE(16);
1125
1126 void __asan_loadN(unsigned long, size_t);
1127 void __asan_loadN_noabort(unsigned long, size_t);
1128 void __asan_storeN(unsigned long, size_t);
1129 void __asan_storeN_noabort(unsigned long, size_t);
1130 void __asan_handle_no_return(void);
1131
1132 void
1133 __asan_loadN(unsigned long addr, size_t size)
1134 {
1135         kasan_shadow_check(addr, size, false, __RET_ADDR);
1136 }
1137
1138 void
1139 __asan_loadN_noabort(unsigned long addr, size_t size)
1140 {
1141         kasan_shadow_check(addr, size, false, __RET_ADDR);
1142 }
1143
1144 void
1145 __asan_storeN(unsigned long addr, size_t size)
1146 {
1147         kasan_shadow_check(addr, size, true, __RET_ADDR);
1148 }
1149
1150 void
1151 __asan_storeN_noabort(unsigned long addr, size_t size)
1152 {
1153         kasan_shadow_check(addr, size, true, __RET_ADDR);
1154 }
1155
1156 void
1157 __asan_handle_no_return(void)
1158 {
1159         /* nothing */
1160 }
1161
1162 #define ASAN_SET_SHADOW(byte) \
1163         void __asan_set_shadow_##byte(void *, size_t);                  \
1164         void __asan_set_shadow_##byte(void *addr, size_t size)          \
1165         {                                                               \
1166                 __builtin_memset((void *)addr, 0x##byte, size);         \
1167         }
1168
1169 ASAN_SET_SHADOW(00);
1170 ASAN_SET_SHADOW(f1);
1171 ASAN_SET_SHADOW(f2);
1172 ASAN_SET_SHADOW(f3);
1173 ASAN_SET_SHADOW(f5);
1174 ASAN_SET_SHADOW(f8);
1175
1176 void __asan_poison_stack_memory(const void *, size_t);
1177 void __asan_unpoison_stack_memory(const void *, size_t);
1178
1179 void
1180 __asan_poison_stack_memory(const void *addr, size_t size)
1181 {
1182         size = roundup(size, KASAN_SHADOW_SCALE);
1183         kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1184 }
1185
1186 void
1187 __asan_unpoison_stack_memory(const void *addr, size_t size)
1188 {
1189         kasan_shadow_Nbyte_markvalid(addr, size);
1190 }
1191
1192 void __asan_alloca_poison(const void *, size_t);
1193 void __asan_allocas_unpoison(const void *, const void *);
1194
1195 void
1196 __asan_alloca_poison(const void *addr, size_t size)
1197 {
1198         const void *l, *r;
1199
1200         KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0,
1201             ("%s: invalid address %p", __func__, addr));
1202
1203         l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1204         r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1205
1206         kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1207         kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1208             KASAN_STACK_MID);
1209         kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1210 }
1211
1212 void
1213 __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1214 {
1215         size_t size;
1216
1217         if (__predict_false(!stkbegin))
1218                 return;
1219         if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1220                 return;
1221         size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1222
1223         kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1224 }
1225
1226 void __asan_poison_memory_region(const void *addr, size_t size);
1227 void __asan_unpoison_memory_region(const void *addr, size_t size);
1228
1229 void
1230 __asan_poison_memory_region(const void *addr, size_t size)
1231 {
1232 }
1233
1234 void
1235 __asan_unpoison_memory_region(const void *addr, size_t size)
1236 {
1237 }