sys/kern/subr_asan.c

   1 /*      $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $      */
   2
   3 /*
   4  * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
   5  * All rights reserved.
   6  *
   7  * This code is part of the KASAN subsystem of the NetBSD kernel.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  23  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  25  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  26  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 #define SAN_RUNTIME
  32
  33 #include <sys/cdefs.h>
  34 #if 0
  35 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $");
  36 #endif
  37
  38 #include <sys/param.h>
  39 #include <sys/systm.h>
  40 #include <sys/asan.h>
  41 #include <sys/kernel.h>
  42 #include <sys/stack.h>
  43 #include <sys/sysctl.h>
  44
  45 #include <machine/asan.h>
  46 #include <machine/bus.h>
  47
  48 /* ASAN constants. Part of the compiler ABI. */
  49 #define KASAN_SHADOW_MASK               (KASAN_SHADOW_SCALE - 1)
  50 #define KASAN_ALLOCA_SCALE_SIZE         32
  51
  52 /* ASAN ABI version. */
  53 #if defined(__clang__) && (__clang_major__ - 0 >= 6)
  54 #define ASAN_ABI_VERSION        8
  55 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
  56 #define ASAN_ABI_VERSION        8
  57 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
  58 #define ASAN_ABI_VERSION        6
  59 #else
  60 #error "Unsupported compiler version"
  61 #endif
  62
  63 #define __RET_ADDR      (unsigned long)__builtin_return_address(0)
  64
  65 /* Global variable descriptor. Part of the compiler ABI.  */
  66 struct __asan_global_source_location {
  67         const char *filename;
  68         int line_no;
  69         int column_no;
  70 };
  71
  72 struct __asan_global {
  73         const void *beg;                /* address of the global variable */
  74         size_t size;                    /* size of the global variable */
  75         size_t size_with_redzone;       /* size with the redzone */
  76         const void *name;               /* name of the variable */
  77         const void *module_name;        /* name of the module where the var is declared */
  78         unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
  79         struct __asan_global_source_location *location;
  80 #if ASAN_ABI_VERSION >= 7
  81         uintptr_t odr_indicator;        /* the address of the ODR indicator symbol */
  82 #endif
  83 };
  84
  85 FEATURE(kasan, "Kernel address sanitizer");
  86
  87 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  88     "KASAN options");
  89
  90 static int panic_on_violation = 1;
  91 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN,
  92     &panic_on_violation, 0,
  93     "Panic if an invalid access is detected");
  94
  95 static bool kasan_enabled __read_mostly = false;
  96
  97 /* -------------------------------------------------------------------------- */
  98
  99 void
 100 kasan_shadow_map(vm_offset_t addr, size_t size)
 101 {
 102         size_t sz, npages, i;
 103         vm_offset_t sva, eva;
 104
 105         KASSERT(addr % KASAN_SHADOW_SCALE == 0,
 106             ("%s: invalid address %#lx", __func__, addr));
 107
 108         sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE;
 109
 110         sva = kasan_md_addr_to_shad(addr);
 111         eva = kasan_md_addr_to_shad(addr) + sz;
 112
 113         sva = rounddown(sva, PAGE_SIZE);
 114         eva = roundup(eva, PAGE_SIZE);
 115
 116         npages = (eva - sva) / PAGE_SIZE;
 117
 118         KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS,
 119             ("%s: invalid address range %#lx-%#lx", __func__, sva, eva));
 120
 121         for (i = 0; i < npages; i++)
 122                 pmap_san_enter(sva + ptoa(i));
 123 }
 124
 125 void
 126 kasan_init(void)
 127 {
 128         int disabled;
 129
 130         disabled = 0;
 131         TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled);
 132         if (disabled)
 133                 return;
 134
 135         /* MD initialization. */
 136         kasan_md_init();
 137
 138         /* Now officially enabled. */
 139         kasan_enabled = true;
 140 }
 141
 142 void
 143 kasan_init_early(vm_offset_t stack, size_t size)
 144 {
 145         kasan_md_init_early(stack, size);
 146 }
 147
 148 static inline const char *
 149 kasan_code_name(uint8_t code)
 150 {
 151         switch (code) {
 152         case KASAN_GENERIC_REDZONE:
 153                 return "GenericRedZone";
 154         case KASAN_MALLOC_REDZONE:
 155                 return "MallocRedZone";
 156         case KASAN_KMEM_REDZONE:
 157                 return "KmemRedZone";
 158         case KASAN_UMA_FREED:
 159                 return "UMAUseAfterFree";
 160         case KASAN_KSTACK_FREED:
 161                 return "KernelStack";
 162         case KASAN_EXEC_ARGS_FREED:
 163                 return "ExecKVA";
 164         case 1 ... 7:
 165                 return "RedZonePartial";
 166         case KASAN_STACK_LEFT:
 167                 return "StackLeft";
 168         case KASAN_STACK_MID:
 169                 return "StackMiddle";
 170         case KASAN_STACK_RIGHT:
 171                 return "StackRight";
 172         case KASAN_USE_AFTER_RET:
 173                 return "UseAfterRet";
 174         case KASAN_USE_AFTER_SCOPE:
 175                 return "UseAfterScope";
 176         default:
 177                 return "Unknown";
 178         }
 179 }
 180
 181 #define REPORT(f, ...) do {                             \
 182         if (panic_on_violation) {                       \
 183                 kasan_enabled = false;                  \
 184                 panic(f, __VA_ARGS__);                  \
 185         } else {                                        \
 186                 struct stack st;                        \
 187                                                         \
 188                 stack_save(&st);                        \
 189                 printf(f "\n", __VA_ARGS__);            \
 190                 stack_print_ddb(&st);                   \
 191         }                                               \
 192 } while (0)
 193
 194 static void
 195 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
 196     uint8_t code)
 197 {
 198         REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)",
 199             size, (write ? "write" : "read"), addr, kasan_code_name(code),
 200             code);
 201 }
 202
 203 static __always_inline void
 204 kasan_shadow_1byte_markvalid(unsigned long addr)
 205 {
 206         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 207         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 208
 209         *byte = last;
 210 }
 211
 212 static __always_inline void
 213 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
 214 {
 215         size_t i;
 216
 217         for (i = 0; i < size; i++) {
 218                 kasan_shadow_1byte_markvalid((unsigned long)addr + i);
 219         }
 220 }
 221
 222 static __always_inline void
 223 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
 224 {
 225         void *shad;
 226
 227         if (__predict_false(size == 0))
 228                 return;
 229         if (__predict_false(kasan_md_unsupported((vm_offset_t)addr)))
 230                 return;
 231
 232         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 233             ("%s: invalid address %p", __func__, addr));
 234         KASSERT(size % KASAN_SHADOW_SCALE == 0,
 235             ("%s: invalid size %zu", __func__, size));
 236
 237         shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr);
 238         size = size >> KASAN_SHADOW_SCALE_SHIFT;
 239
 240         __builtin_memset(shad, code, size);
 241 }
 242
 243 /*
 244  * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
 245  * and the rest as invalid. There are generally two use cases:
 246  *
 247  *  o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
 248  *    the redzone at the end of the buffer as invalid. If the entire is to be
 249  *    marked invalid, origsize will be 0.
 250  *
 251  *  o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
 252  */
 253 void
 254 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
 255 {
 256         size_t i, n, redz;
 257         int8_t *shad;
 258
 259         if (__predict_false(!kasan_enabled))
 260                 return;
 261
 262         if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS &&
 263             (vm_offset_t)addr < DMAP_MAX_ADDRESS)
 264                 return;
 265
 266         KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS &&
 267             (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS,
 268             ("%s: invalid address %p", __func__, addr));
 269         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 270             ("%s: invalid address %p", __func__, addr));
 271         redz = redzsize - roundup(size, KASAN_SHADOW_SCALE);
 272         KASSERT(redz % KASAN_SHADOW_SCALE == 0,
 273             ("%s: invalid size %zu", __func__, redz));
 274         shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr);
 275
 276         /* Chunks of 8 bytes, valid. */
 277         n = size / KASAN_SHADOW_SCALE;
 278         for (i = 0; i < n; i++) {
 279                 *shad++ = 0;
 280         }
 281
 282         /* Possibly one chunk, mid. */
 283         if ((size & KASAN_SHADOW_MASK) != 0) {
 284                 *shad++ = (size & KASAN_SHADOW_MASK);
 285         }
 286
 287         /* Chunks of 8 bytes, invalid. */
 288         n = redz / KASAN_SHADOW_SCALE;
 289         for (i = 0; i < n; i++) {
 290                 *shad++ = code;
 291         }
 292 }
 293
 294 /* -------------------------------------------------------------------------- */
 295
 296 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size)                 \
 297         (addr >> KASAN_SHADOW_SCALE_SHIFT) !=                   \
 298             ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
 299
 300 static __always_inline bool
 301 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
 302 {
 303         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 304         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 305
 306         if (__predict_true(*byte == 0 || last <= *byte)) {
 307                 return (true);
 308         }
 309         *code = *byte;
 310         return (false);
 311 }
 312
 313 static __always_inline bool
 314 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
 315 {
 316         int8_t *byte, last;
 317
 318         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
 319                 return (kasan_shadow_1byte_isvalid(addr, code) &&
 320                     kasan_shadow_1byte_isvalid(addr+1, code));
 321         }
 322
 323         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 324         last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
 325
 326         if (__predict_true(*byte == 0 || last <= *byte)) {
 327                 return (true);
 328         }
 329         *code = *byte;
 330         return (false);
 331 }
 332
 333 static __always_inline bool
 334 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
 335 {
 336         int8_t *byte, last;
 337
 338         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
 339                 return (kasan_shadow_2byte_isvalid(addr, code) &&
 340                     kasan_shadow_2byte_isvalid(addr+2, code));
 341         }
 342
 343         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 344         last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
 345
 346         if (__predict_true(*byte == 0 || last <= *byte)) {
 347                 return (true);
 348         }
 349         *code = *byte;
 350         return (false);
 351 }
 352
 353 static __always_inline bool
 354 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
 355 {
 356         int8_t *byte, last;
 357
 358         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
 359                 return (kasan_shadow_4byte_isvalid(addr, code) &&
 360                     kasan_shadow_4byte_isvalid(addr+4, code));
 361         }
 362
 363         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 364         last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
 365
 366         if (__predict_true(*byte == 0 || last <= *byte)) {
 367                 return (true);
 368         }
 369         *code = *byte;
 370         return (false);
 371 }
 372
 373 static __always_inline bool
 374 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
 375 {
 376         size_t i;
 377
 378         for (i = 0; i < size; i++) {
 379                 if (!kasan_shadow_1byte_isvalid(addr+i, code))
 380                         return (false);
 381         }
 382
 383         return (true);
 384 }
 385
 386 static __always_inline void
 387 kasan_shadow_check(unsigned long addr, size_t size, bool write,
 388     unsigned long retaddr)
 389 {
 390         uint8_t code;
 391         bool valid;
 392
 393         if (__predict_false(!kasan_enabled))
 394                 return;
 395         if (__predict_false(size == 0))
 396                 return;
 397         if (__predict_false(kasan_md_unsupported(addr)))
 398                 return;
 399         if (KERNEL_PANICKED())
 400                 return;
 401
 402         if (__builtin_constant_p(size)) {
 403                 switch (size) {
 404                 case 1:
 405                         valid = kasan_shadow_1byte_isvalid(addr, &code);
 406                         break;
 407                 case 2:
 408                         valid = kasan_shadow_2byte_isvalid(addr, &code);
 409                         break;
 410                 case 4:
 411                         valid = kasan_shadow_4byte_isvalid(addr, &code);
 412                         break;
 413                 case 8:
 414                         valid = kasan_shadow_8byte_isvalid(addr, &code);
 415                         break;
 416                 default:
 417                         valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 418                         break;
 419                 }
 420         } else {
 421                 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 422         }
 423
 424         if (__predict_false(!valid)) {
 425                 kasan_report(addr, size, write, retaddr, code);
 426         }
 427 }
 428
 429 /* -------------------------------------------------------------------------- */
 430
 431 void *
 432 kasan_memcpy(void *dst, const void *src, size_t len)
 433 {
 434         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 435         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 436         return (__builtin_memcpy(dst, src, len));
 437 }
 438
 439 int
 440 kasan_memcmp(const void *b1, const void *b2, size_t len)
 441 {
 442         kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
 443         kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
 444         return (__builtin_memcmp(b1, b2, len));
 445 }
 446
 447 void *
 448 kasan_memset(void *b, int c, size_t len)
 449 {
 450         kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
 451         return (__builtin_memset(b, c, len));
 452 }
 453
 454 void *
 455 kasan_memmove(void *dst, const void *src, size_t len)
 456 {
 457         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 458         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 459         return (__builtin_memmove(dst, src, len));
 460 }
 461
 462 size_t
 463 kasan_strlen(const char *str)
 464 {
 465         const char *s;
 466
 467         s = str;
 468         while (1) {
 469                 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
 470                 if (*s == '\0')
 471                         break;
 472                 s++;
 473         }
 474
 475         return (s - str);
 476 }
 477
 478 char *
 479 kasan_strcpy(char *dst, const char *src)
 480 {
 481         char *save = dst;
 482
 483         while (1) {
 484                 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
 485                 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
 486                 *dst = *src;
 487                 if (*src == '\0')
 488                         break;
 489                 src++, dst++;
 490         }
 491
 492         return save;
 493 }
 494
 495 int
 496 kasan_strcmp(const char *s1, const char *s2)
 497 {
 498         while (1) {
 499                 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
 500                 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
 501                 if (*s1 != *s2)
 502                         break;
 503                 if (*s1 == '\0')
 504                         return 0;
 505                 s1++, s2++;
 506         }
 507
 508         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 509 }
 510
 511 int
 512 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
 513 {
 514         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 515         return (copyin(uaddr, kaddr, len));
 516 }
 517
 518 int
 519 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 520 {
 521         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 522         return (copyinstr(uaddr, kaddr, len, done));
 523 }
 524
 525 int
 526 kasan_copyout(const void *kaddr, void *uaddr, size_t len)
 527 {
 528         kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
 529         return (copyout(kaddr, uaddr, len));
 530 }
 531
 532 /* -------------------------------------------------------------------------- */
 533
 534 int
 535 kasan_fubyte(volatile const void *base)
 536 {
 537         return (fubyte(base));
 538 }
 539
 540 int
 541 kasan_fuword16(volatile const void *base)
 542 {
 543         return (fuword16(base));
 544 }
 545
 546 int
 547 kasan_fueword(volatile const void *base, long *val)
 548 {
 549         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 550         return (fueword(base, val));
 551 }
 552
 553 int
 554 kasan_fueword32(volatile const void *base, int32_t *val)
 555 {
 556         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 557         return (fueword32(base, val));
 558 }
 559
 560 int
 561 kasan_fueword64(volatile const void *base, int64_t *val)
 562 {
 563         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 564         return (fueword64(base, val));
 565 }
 566
 567 int
 568 kasan_subyte(volatile void *base, int byte)
 569 {
 570         return (subyte(base, byte));
 571 }
 572
 573 int
 574 kasan_suword(volatile void *base, long word)
 575 {
 576         return (suword(base, word));
 577 }
 578
 579 int
 580 kasan_suword16(volatile void *base, int word)
 581 {
 582         return (suword16(base, word));
 583 }
 584
 585 int
 586 kasan_suword32(volatile void *base, int32_t word)
 587 {
 588         return (suword32(base, word));
 589 }
 590
 591 int
 592 kasan_suword64(volatile void *base, int64_t word)
 593 {
 594         return (suword64(base, word));
 595 }
 596
 597 int
 598 kasan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
 599     uint32_t newval)
 600 {
 601         kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
 602             __RET_ADDR);
 603         return (casueword32(base, oldval, oldvalp, newval));
 604 }
 605
 606 int
 607 kasan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
 608     u_long newval)
 609 {
 610         kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
 611             __RET_ADDR);
 612         return (casueword(base, oldval, oldvalp, newval));
 613 }
 614
 615 /* -------------------------------------------------------------------------- */
 616
 617 #include <machine/atomic.h>
 618 #include <sys/atomic_san.h>
 619
 620 #define _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 621         void kasan_atomic_add_##name(volatile type *ptr, type val)      \
 622         {                                                               \
 623                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 624                     __RET_ADDR);                                        \
 625                 atomic_add_##name(ptr, val);                            \
 626         }
 627
 628 #define ASAN_ATOMIC_FUNC_ADD(name, type)                                \
 629         _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 630         _ASAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
 631         _ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
 632
 633 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 634         void kasan_atomic_subtract_##name(volatile type *ptr, type val) \
 635         {                                                               \
 636                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 637                     __RET_ADDR);                                        \
 638                 atomic_subtract_##name(ptr, val);                       \
 639         }
 640
 641 #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
 642         _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 643         _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
 644         _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
 645
 646 #define _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 647         void kasan_atomic_set_##name(volatile type *ptr, type val)      \
 648         {                                                               \
 649                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 650                     __RET_ADDR);                                        \
 651                 atomic_set_##name(ptr, val);                            \
 652         }
 653
 654 #define ASAN_ATOMIC_FUNC_SET(name, type)                                \
 655         _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 656         _ASAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
 657         _ASAN_ATOMIC_FUNC_SET(rel_##name, type)
 658
 659 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 660         void kasan_atomic_clear_##name(volatile type *ptr, type val)    \
 661         {                                                               \
 662                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 663                     __RET_ADDR);                                        \
 664                 atomic_clear_##name(ptr, val);                          \
 665         }
 666
 667 #define ASAN_ATOMIC_FUNC_CLEAR(name, type)                              \
 668         _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 669         _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
 670         _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
 671
 672 #define ASAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
 673         type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \
 674         {                                                               \
 675                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 676                     __RET_ADDR);                                        \
 677                 return (atomic_fetchadd_##name(ptr, val));              \
 678         }
 679
 680 #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
 681         type kasan_atomic_readandclear_##name(volatile type *ptr)       \
 682         {                                                               \
 683                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 684                     __RET_ADDR);                                        \
 685                 return (atomic_readandclear_##name(ptr));               \
 686         }
 687
 688 #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
 689         int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
 690         {                                                               \
 691                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 692                     __RET_ADDR);                                        \
 693                 return (atomic_testandclear_##name(ptr, v));            \
 694         }
 695
 696 #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
 697         int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
 698         {                                                               \
 699                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 700                     __RET_ADDR);                                        \
 701                 return (atomic_testandset_##name(ptr, v));              \
 702         }
 703
 704 #define ASAN_ATOMIC_FUNC_SWAP(name, type)                               \
 705         type kasan_atomic_swap_##name(volatile type *ptr, type val)     \
 706         {                                                               \
 707                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 708                     __RET_ADDR);                                        \
 709                 return (atomic_swap_##name(ptr, val));                  \
 710         }
 711
 712 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 713         int kasan_atomic_cmpset_##name(volatile type *ptr, type oval,   \
 714             type nval)                                                  \
 715         {                                                               \
 716                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 717                     __RET_ADDR);                                        \
 718                 return (atomic_cmpset_##name(ptr, oval, nval));         \
 719         }
 720
 721 #define ASAN_ATOMIC_FUNC_CMPSET(name, type)                             \
 722         _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 723         _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
 724         _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
 725
 726 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 727         int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
 728             type nval)                                                  \
 729         {                                                               \
 730                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 731                     __RET_ADDR);                                        \
 732                 return (atomic_fcmpset_##name(ptr, oval, nval));        \
 733         }
 734
 735 #define ASAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
 736         _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 737         _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
 738         _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
 739
 740 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
 741         void kasan_atomic_thread_fence_##name(void)                     \
 742         {                                                               \
 743                 atomic_thread_fence_##name();                           \
 744         }
 745
 746 #define _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 747         type kasan_atomic_load_##name(volatile type *ptr)               \
 748         {                                                               \
 749                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 750                     __RET_ADDR);                                        \
 751                 return (atomic_load_##name(ptr));                       \
 752         }
 753
 754 #define ASAN_ATOMIC_FUNC_LOAD(name, type)                               \
 755         _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 756         _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
 757
 758 #define _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 759         void kasan_atomic_store_##name(volatile type *ptr, type val)    \
 760         {                                                               \
 761                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 762                     __RET_ADDR);                                        \
 763                 atomic_store_##name(ptr, val);                          \
 764         }
 765
 766 #define ASAN_ATOMIC_FUNC_STORE(name, type)                              \
 767         _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 768         _ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
 769
 770 ASAN_ATOMIC_FUNC_ADD(8, uint8_t);
 771 ASAN_ATOMIC_FUNC_ADD(16, uint16_t);
 772 ASAN_ATOMIC_FUNC_ADD(32, uint32_t);
 773 ASAN_ATOMIC_FUNC_ADD(64, uint64_t);
 774 ASAN_ATOMIC_FUNC_ADD(int, u_int);
 775 ASAN_ATOMIC_FUNC_ADD(long, u_long);
 776 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
 777
 778 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
 779 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
 780 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
 781 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
 782 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
 783 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
 784 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
 785
 786 ASAN_ATOMIC_FUNC_SET(8, uint8_t);
 787 ASAN_ATOMIC_FUNC_SET(16, uint16_t);
 788 ASAN_ATOMIC_FUNC_SET(32, uint32_t);
 789 ASAN_ATOMIC_FUNC_SET(64, uint64_t);
 790 ASAN_ATOMIC_FUNC_SET(int, u_int);
 791 ASAN_ATOMIC_FUNC_SET(long, u_long);
 792 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
 793
 794 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
 795 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
 796 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
 797 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
 798 ASAN_ATOMIC_FUNC_CLEAR(int, u_int);
 799 ASAN_ATOMIC_FUNC_CLEAR(long, u_long);
 800 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
 801
 802 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
 803 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
 804 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int);
 805 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long);
 806
 807 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
 808 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
 809 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
 810 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
 811 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
 812
 813 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
 814 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
 815 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
 816 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
 817
 818 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
 819 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
 820 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
 821 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
 822
 823 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t);
 824 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t);
 825 ASAN_ATOMIC_FUNC_SWAP(int, u_int);
 826 ASAN_ATOMIC_FUNC_SWAP(long, u_long);
 827 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
 828
 829 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
 830 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
 831 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
 832 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
 833 ASAN_ATOMIC_FUNC_CMPSET(int, u_int);
 834 ASAN_ATOMIC_FUNC_CMPSET(long, u_long);
 835 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
 836
 837 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
 838 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
 839 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
 840 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
 841 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int);
 842 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long);
 843 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
 844
 845 _ASAN_ATOMIC_FUNC_LOAD(bool, bool);
 846 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t);
 847 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t);
 848 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t);
 849 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t);
 850 ASAN_ATOMIC_FUNC_LOAD(char, u_char);
 851 ASAN_ATOMIC_FUNC_LOAD(short, u_short);
 852 ASAN_ATOMIC_FUNC_LOAD(int, u_int);
 853 ASAN_ATOMIC_FUNC_LOAD(long, u_long);
 854 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
 855
 856 _ASAN_ATOMIC_FUNC_STORE(bool, bool);
 857 ASAN_ATOMIC_FUNC_STORE(8, uint8_t);
 858 ASAN_ATOMIC_FUNC_STORE(16, uint16_t);
 859 ASAN_ATOMIC_FUNC_STORE(32, uint32_t);
 860 ASAN_ATOMIC_FUNC_STORE(64, uint64_t);
 861 ASAN_ATOMIC_FUNC_STORE(char, u_char);
 862 ASAN_ATOMIC_FUNC_STORE(short, u_short);
 863 ASAN_ATOMIC_FUNC_STORE(int, u_int);
 864 ASAN_ATOMIC_FUNC_STORE(long, u_long);
 865 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
 866
 867 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq);
 868 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel);
 869 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
 870 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
 871
 872 void
 873 kasan_atomic_interrupt_fence(void)
 874 {
 875 }
 876
 877 /* -------------------------------------------------------------------------- */
 878
 879 #include <sys/bus.h>
 880 #include <machine/bus.h>
 881 #include <sys/bus_san.h>
 882
 883 int
 884 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
 885     int flags, bus_space_handle_t *handlep)
 886 {
 887         return (bus_space_map(tag, hnd, size, flags, handlep));
 888 }
 889
 890 void
 891 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
 892     bus_size_t size)
 893 {
 894         bus_space_unmap(tag, hnd, size);
 895 }
 896
 897 int
 898 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
 899     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
 900 {
 901         return (bus_space_subregion(tag, hnd, offset, size, handlep));
 902 }
 903
 904 void
 905 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
 906     bus_size_t size)
 907 {
 908         bus_space_free(tag, hnd, size);
 909 }
 910
 911 void
 912 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
 913     bus_size_t offset, bus_size_t size, int flags)
 914 {
 915         bus_space_barrier(tag, hnd, offset, size, flags);
 916 }
 917
 918 #define ASAN_BUS_READ_FUNC(func, width, type)                           \
 919         type kasan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
 920             bus_space_handle_t hnd, bus_size_t offset)                  \
 921         {                                                               \
 922                 return (bus_space_read##func##_##width(tag, hnd,        \
 923                     offset));                                           \
 924         }                                                               \
 925
 926 #define ASAN_BUS_READ_PTR_FUNC(func, width, type)                       \
 927         void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
 928             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
 929             bus_size_t count)                                           \
 930         {                                                               \
 931                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 932                     false, __RET_ADDR);                                 \
 933                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
 934                     count);                                             \
 935         }
 936
 937 ASAN_BUS_READ_FUNC(, 1, uint8_t)
 938 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t)
 939 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
 940 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
 941 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
 942 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
 943
 944 ASAN_BUS_READ_FUNC(, 2, uint16_t)
 945 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t)
 946 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
 947 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
 948 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
 949 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
 950
 951 ASAN_BUS_READ_FUNC(, 4, uint32_t)
 952 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t)
 953 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
 954 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
 955 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
 956 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
 957
 958 ASAN_BUS_READ_FUNC(, 8, uint64_t)
 959 #if defined(__aarch64__)
 960 ASAN_BUS_READ_FUNC(_stream, 8, uint64_t)
 961 ASAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
 962 ASAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
 963 ASAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
 964 ASAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
 965 #endif
 966
 967 #define ASAN_BUS_WRITE_FUNC(func, width, type)                          \
 968         void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \
 969             bus_space_handle_t hnd, bus_size_t offset, type value)      \
 970         {                                                               \
 971                 bus_space_write##func##_##width(tag, hnd, offset, value);\
 972         }                                                               \
 973
 974 #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
 975         void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
 976             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
 977             bus_size_t count)                                           \
 978         {                                                               \
 979                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 980                     true, __RET_ADDR);                                  \
 981                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
 982                     count);                                             \
 983         }
 984
 985 ASAN_BUS_WRITE_FUNC(, 1, uint8_t)
 986 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
 987 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
 988 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
 989 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
 990 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
 991
 992 ASAN_BUS_WRITE_FUNC(, 2, uint16_t)
 993 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
 994 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
 995 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
 996 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
 997 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
 998
 999 ASAN_BUS_WRITE_FUNC(, 4, uint32_t)
1000 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1001 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1002 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1003 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1004 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1005
1006 ASAN_BUS_WRITE_FUNC(, 8, uint64_t)
1007
1008 #define ASAN_BUS_SET_FUNC(func, width, type)                            \
1009         void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
1010             bus_space_handle_t hnd, bus_size_t offset, type value,      \
1011             bus_size_t count)                                           \
1012         {                                                               \
1013                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1014                     count);                                             \
1015         }
1016
1017 ASAN_BUS_SET_FUNC(multi, 1, uint8_t)
1018 ASAN_BUS_SET_FUNC(region, 1, uint8_t)
1019 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1020 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1021
1022 ASAN_BUS_SET_FUNC(multi, 2, uint16_t)
1023 ASAN_BUS_SET_FUNC(region, 2, uint16_t)
1024 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1025 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1026
1027 ASAN_BUS_SET_FUNC(multi, 4, uint32_t)
1028 ASAN_BUS_SET_FUNC(region, 4, uint32_t)
1029 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1030 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1031
1032 #define ASAN_BUS_PEEK_FUNC(width, type)                                 \
1033         int kasan_bus_space_peek_##width(bus_space_tag_t tag,           \
1034             bus_space_handle_t hnd, bus_size_t offset, type *valuep)    \
1035         {                                                               \
1036                 return (bus_space_peek_##width(tag, hnd, offset,        \
1037                     valuep));                                           \
1038         }
1039
1040 ASAN_BUS_PEEK_FUNC(1, uint8_t)
1041 ASAN_BUS_PEEK_FUNC(2, uint16_t)
1042 ASAN_BUS_PEEK_FUNC(4, uint32_t)
1043 ASAN_BUS_PEEK_FUNC(8, uint64_t)
1044
1045 #define ASAN_BUS_POKE_FUNC(width, type)                                 \
1046         int kasan_bus_space_poke_##width(bus_space_tag_t tag,           \
1047             bus_space_handle_t hnd, bus_size_t offset, type value)      \
1048         {                                                               \
1049                 return (bus_space_poke_##width(tag, hnd, offset,        \
1050                     value));                                            \
1051         }
1052
1053 ASAN_BUS_POKE_FUNC(1, uint8_t)
1054 ASAN_BUS_POKE_FUNC(2, uint16_t)
1055 ASAN_BUS_POKE_FUNC(4, uint32_t)
1056 ASAN_BUS_POKE_FUNC(8, uint64_t)
1057
1058 /* -------------------------------------------------------------------------- */
1059
1060 void __asan_register_globals(struct __asan_global *, size_t);
1061 void __asan_unregister_globals(struct __asan_global *, size_t);
1062
1063 void
1064 __asan_register_globals(struct __asan_global *globals, size_t n)
1065 {
1066         size_t i;
1067
1068         for (i = 0; i < n; i++) {
1069                 kasan_mark(globals[i].beg, globals[i].size,
1070                     globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
1071         }
1072 }
1073
1074 void
1075 __asan_unregister_globals(struct __asan_global *globals, size_t n)
1076 {
1077         size_t i;
1078
1079         for (i = 0; i < n; i++) {
1080                 kasan_mark(globals[i].beg, globals[i].size_with_redzone,
1081                     globals[i].size_with_redzone, 0);
1082         }
1083 }
1084
1085 #define ASAN_LOAD_STORE(size)                                   \
1086         void __asan_load##size(unsigned long);                  \
1087         void __asan_load##size(unsigned long addr)              \
1088         {                                                       \
1089                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1090         }                                                       \
1091         void __asan_load##size##_noabort(unsigned long);        \
1092         void __asan_load##size##_noabort(unsigned long addr)    \
1093         {                                                       \
1094                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1095         }                                                       \
1096         void __asan_store##size(unsigned long);                 \
1097         void __asan_store##size(unsigned long addr)             \
1098         {                                                       \
1099                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1100         }                                                       \
1101         void __asan_store##size##_noabort(unsigned long);       \
1102         void __asan_store##size##_noabort(unsigned long addr)   \
1103         {                                                       \
1104                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1105         }
1106
1107 ASAN_LOAD_STORE(1);
1108 ASAN_LOAD_STORE(2);
1109 ASAN_LOAD_STORE(4);
1110 ASAN_LOAD_STORE(8);
1111 ASAN_LOAD_STORE(16);
1112
1113 void __asan_loadN(unsigned long, size_t);
1114 void __asan_loadN_noabort(unsigned long, size_t);
1115 void __asan_storeN(unsigned long, size_t);
1116 void __asan_storeN_noabort(unsigned long, size_t);
1117 void __asan_handle_no_return(void);
1118
1119 void
1120 __asan_loadN(unsigned long addr, size_t size)
1121 {
1122         kasan_shadow_check(addr, size, false, __RET_ADDR);
1123 }
1124
1125 void
1126 __asan_loadN_noabort(unsigned long addr, size_t size)
1127 {
1128         kasan_shadow_check(addr, size, false, __RET_ADDR);
1129 }
1130
1131 void
1132 __asan_storeN(unsigned long addr, size_t size)
1133 {
1134         kasan_shadow_check(addr, size, true, __RET_ADDR);
1135 }
1136
1137 void
1138 __asan_storeN_noabort(unsigned long addr, size_t size)
1139 {
1140         kasan_shadow_check(addr, size, true, __RET_ADDR);
1141 }
1142
1143 void
1144 __asan_handle_no_return(void)
1145 {
1146         /* nothing */
1147 }
1148
1149 #define ASAN_SET_SHADOW(byte) \
1150         void __asan_set_shadow_##byte(void *, size_t);                  \
1151         void __asan_set_shadow_##byte(void *addr, size_t size)          \
1152         {                                                               \
1153                 __builtin_memset((void *)addr, 0x##byte, size);         \
1154         }
1155
1156 ASAN_SET_SHADOW(00);
1157 ASAN_SET_SHADOW(f1);
1158 ASAN_SET_SHADOW(f2);
1159 ASAN_SET_SHADOW(f3);
1160 ASAN_SET_SHADOW(f5);
1161 ASAN_SET_SHADOW(f8);
1162
1163 void __asan_poison_stack_memory(const void *, size_t);
1164 void __asan_unpoison_stack_memory(const void *, size_t);
1165
1166 void
1167 __asan_poison_stack_memory(const void *addr, size_t size)
1168 {
1169         size = roundup(size, KASAN_SHADOW_SCALE);
1170         kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1171 }
1172
1173 void
1174 __asan_unpoison_stack_memory(const void *addr, size_t size)
1175 {
1176         kasan_shadow_Nbyte_markvalid(addr, size);
1177 }
1178
1179 void __asan_alloca_poison(const void *, size_t);
1180 void __asan_allocas_unpoison(const void *, const void *);
1181
1182 void
1183 __asan_alloca_poison(const void *addr, size_t size)
1184 {
1185         const void *l, *r;
1186
1187         KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0,
1188             ("%s: invalid address %p", __func__, addr));
1189
1190         l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1191         r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1192
1193         kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1194         kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1195             KASAN_STACK_MID);
1196         kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1197 }
1198
1199 void
1200 __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1201 {
1202         size_t size;
1203
1204         if (__predict_false(!stkbegin))
1205                 return;
1206         if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1207                 return;
1208         size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1209
1210         kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1211 }
1212
1213 void __asan_poison_memory_region(const void *addr, size_t size);
1214 void __asan_unpoison_memory_region(const void *addr, size_t size);
1215
1216 void
1217 __asan_poison_memory_region(const void *addr, size_t size)
1218 {
1219 }
1220
1221 void
1222 __asan_unpoison_memory_region(const void *addr, size_t size)
1223 {
1224 }