sys/kern/subr_asan.c

   1 /*      $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $      */
   2
   3 /*
   4  * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
   5  * All rights reserved.
   6  *
   7  * This code is part of the KASAN subsystem of the NetBSD kernel.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  23  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  25  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  26  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 #define SAN_RUNTIME
  32
  33 #include <sys/cdefs.h>
  34 #if 0
  35 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $");
  36 #endif
  37
  38 #include <sys/param.h>
  39 #include <sys/systm.h>
  40 #include <sys/asan.h>
  41 #include <sys/kernel.h>
  42 #include <sys/stack.h>
  43 #include <sys/sysctl.h>
  44
  45 #include <machine/asan.h>
  46 #include <machine/bus.h>
  47
  48 /* ASAN constants. Part of the compiler ABI. */
  49 #define KASAN_SHADOW_MASK               (KASAN_SHADOW_SCALE - 1)
  50 #define KASAN_ALLOCA_SCALE_SIZE         32
  51
  52 /* ASAN ABI version. */
  53 #if defined(__clang__) && (__clang_major__ - 0 >= 6)
  54 #define ASAN_ABI_VERSION        8
  55 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
  56 #define ASAN_ABI_VERSION        8
  57 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
  58 #define ASAN_ABI_VERSION        6
  59 #else
  60 #error "Unsupported compiler version"
  61 #endif
  62
  63 #define __RET_ADDR      (unsigned long)__builtin_return_address(0)
  64
  65 /* Global variable descriptor. Part of the compiler ABI.  */
  66 struct __asan_global_source_location {
  67         const char *filename;
  68         int line_no;
  69         int column_no;
  70 };
  71
  72 struct __asan_global {
  73         const void *beg;                /* address of the global variable */
  74         size_t size;                    /* size of the global variable */
  75         size_t size_with_redzone;       /* size with the redzone */
  76         const void *name;               /* name of the variable */
  77         const void *module_name;        /* name of the module where the var is declared */
  78         unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
  79         struct __asan_global_source_location *location;
  80 #if ASAN_ABI_VERSION >= 7
  81         uintptr_t odr_indicator;        /* the address of the ODR indicator symbol */
  82 #endif
  83 };
  84
  85 FEATURE(kasan, "Kernel address sanitizer");
  86
  87 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  88     "KASAN options");
  89
  90 static int panic_on_violation = 1;
  91 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN,
  92     &panic_on_violation, 0,
  93     "Panic if an invalid access is detected");
  94
  95 #define kasan_enabled (!kasan_disabled)
  96 static bool kasan_disabled __read_mostly = true;
  97 SYSCTL_BOOL(_debug_kasan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
  98     &kasan_disabled, 0, "KASAN is disabled");
  99
 100 /* -------------------------------------------------------------------------- */
 101
 102 void
 103 kasan_shadow_map(vm_offset_t addr, size_t size)
 104 {
 105         size_t sz, npages, i;
 106         vm_offset_t sva, eva;
 107
 108         KASSERT(addr % KASAN_SHADOW_SCALE == 0,
 109             ("%s: invalid address %#lx", __func__, addr));
 110
 111         sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE;
 112
 113         sva = kasan_md_addr_to_shad(addr);
 114         eva = kasan_md_addr_to_shad(addr) + sz;
 115
 116         sva = rounddown(sva, PAGE_SIZE);
 117         eva = roundup(eva, PAGE_SIZE);
 118
 119         npages = (eva - sva) / PAGE_SIZE;
 120
 121         KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS,
 122             ("%s: invalid address range %#lx-%#lx", __func__, sva, eva));
 123
 124         for (i = 0; i < npages; i++)
 125                 pmap_kasan_enter(sva + ptoa(i));
 126 }
 127
 128 void
 129 kasan_init(void)
 130 {
 131         int disabled;
 132
 133         disabled = 0;
 134         TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled);
 135         if (disabled)
 136                 return;
 137
 138         /* MD initialization. */
 139         kasan_md_init();
 140
 141         /* Now officially enabled. */
 142         kasan_disabled = false;
 143 }
 144
 145 static inline const char *
 146 kasan_code_name(uint8_t code)
 147 {
 148         switch (code) {
 149         case KASAN_GENERIC_REDZONE:
 150                 return "GenericRedZone";
 151         case KASAN_MALLOC_REDZONE:
 152                 return "MallocRedZone";
 153         case KASAN_KMEM_REDZONE:
 154                 return "KmemRedZone";
 155         case KASAN_UMA_FREED:
 156                 return "UMAUseAfterFree";
 157         case KASAN_KSTACK_FREED:
 158                 return "KernelStack";
 159         case KASAN_EXEC_ARGS_FREED:
 160                 return "ExecKVA";
 161         case 1 ... 7:
 162                 return "RedZonePartial";
 163         case KASAN_STACK_LEFT:
 164                 return "StackLeft";
 165         case KASAN_STACK_MID:
 166                 return "StackMiddle";
 167         case KASAN_STACK_RIGHT:
 168                 return "StackRight";
 169         case KASAN_USE_AFTER_RET:
 170                 return "UseAfterRet";
 171         case KASAN_USE_AFTER_SCOPE:
 172                 return "UseAfterScope";
 173         default:
 174                 return "Unknown";
 175         }
 176 }
 177
 178 #define REPORT(f, ...) do {                             \
 179         if (panic_on_violation) {                       \
 180                 kasan_disabled = true;                  \
 181                 panic(f, __VA_ARGS__);                  \
 182         } else {                                        \
 183                 struct stack st;                        \
 184                                                         \
 185                 stack_save(&st);                        \
 186                 printf(f "\n", __VA_ARGS__);            \
 187                 stack_print_ddb(&st);                   \
 188         }                                               \
 189 } while (0)
 190
 191 static void
 192 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
 193     uint8_t code)
 194 {
 195         REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)",
 196             size, (write ? "write" : "read"), addr, kasan_code_name(code),
 197             code);
 198 }
 199
 200 static __always_inline void
 201 kasan_shadow_1byte_markvalid(unsigned long addr)
 202 {
 203         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 204         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 205
 206         *byte = last;
 207 }
 208
 209 static __always_inline void
 210 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
 211 {
 212         size_t i;
 213
 214         for (i = 0; i < size; i++) {
 215                 kasan_shadow_1byte_markvalid((unsigned long)addr + i);
 216         }
 217 }
 218
 219 static __always_inline void
 220 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
 221 {
 222         void *shad;
 223
 224         if (__predict_false(size == 0))
 225                 return;
 226         if (__predict_false(kasan_md_unsupported((vm_offset_t)addr)))
 227                 return;
 228
 229         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 230             ("%s: invalid address %p", __func__, addr));
 231         KASSERT(size % KASAN_SHADOW_SCALE == 0,
 232             ("%s: invalid size %zu", __func__, size));
 233
 234         shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr);
 235         size = size >> KASAN_SHADOW_SCALE_SHIFT;
 236
 237         __builtin_memset(shad, code, size);
 238 }
 239
 240 /*
 241  * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
 242  * and the rest as invalid. There are generally two use cases:
 243  *
 244  *  o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
 245  *    the redzone at the end of the buffer as invalid. If the entire is to be
 246  *    marked invalid, origsize will be 0.
 247  *
 248  *  o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
 249  */
 250 void
 251 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
 252 {
 253         size_t i, n, redz;
 254         int8_t *shad;
 255
 256         if (__predict_false(!kasan_enabled))
 257                 return;
 258
 259         if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS &&
 260             (vm_offset_t)addr < DMAP_MAX_ADDRESS)
 261                 return;
 262
 263         KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS &&
 264             (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS,
 265             ("%s: invalid address %p", __func__, addr));
 266         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 267             ("%s: invalid address %p", __func__, addr));
 268         redz = redzsize - roundup(size, KASAN_SHADOW_SCALE);
 269         KASSERT(redz % KASAN_SHADOW_SCALE == 0,
 270             ("%s: invalid size %zu", __func__, redz));
 271         shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr);
 272
 273         /* Chunks of 8 bytes, valid. */
 274         n = size / KASAN_SHADOW_SCALE;
 275         for (i = 0; i < n; i++) {
 276                 *shad++ = 0;
 277         }
 278
 279         /* Possibly one chunk, mid. */
 280         if ((size & KASAN_SHADOW_MASK) != 0) {
 281                 *shad++ = (size & KASAN_SHADOW_MASK);
 282         }
 283
 284         /* Chunks of 8 bytes, invalid. */
 285         n = redz / KASAN_SHADOW_SCALE;
 286         for (i = 0; i < n; i++) {
 287                 *shad++ = code;
 288         }
 289 }
 290
 291 /* -------------------------------------------------------------------------- */
 292
 293 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size)                 \
 294         (addr >> KASAN_SHADOW_SCALE_SHIFT) !=                   \
 295             ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
 296
 297 static __always_inline bool
 298 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
 299 {
 300         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 301         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 302
 303         if (__predict_true(*byte == 0 || last <= *byte)) {
 304                 return (true);
 305         }
 306         *code = *byte;
 307         return (false);
 308 }
 309
 310 static __always_inline bool
 311 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
 312 {
 313         int8_t *byte, last;
 314
 315         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
 316                 return (kasan_shadow_1byte_isvalid(addr, code) &&
 317                     kasan_shadow_1byte_isvalid(addr+1, code));
 318         }
 319
 320         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 321         last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
 322
 323         if (__predict_true(*byte == 0 || last <= *byte)) {
 324                 return (true);
 325         }
 326         *code = *byte;
 327         return (false);
 328 }
 329
 330 static __always_inline bool
 331 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
 332 {
 333         int8_t *byte, last;
 334
 335         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
 336                 return (kasan_shadow_2byte_isvalid(addr, code) &&
 337                     kasan_shadow_2byte_isvalid(addr+2, code));
 338         }
 339
 340         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 341         last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
 342
 343         if (__predict_true(*byte == 0 || last <= *byte)) {
 344                 return (true);
 345         }
 346         *code = *byte;
 347         return (false);
 348 }
 349
 350 static __always_inline bool
 351 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
 352 {
 353         int8_t *byte, last;
 354
 355         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
 356                 return (kasan_shadow_4byte_isvalid(addr, code) &&
 357                     kasan_shadow_4byte_isvalid(addr+4, code));
 358         }
 359
 360         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 361         last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
 362
 363         if (__predict_true(*byte == 0 || last <= *byte)) {
 364                 return (true);
 365         }
 366         *code = *byte;
 367         return (false);
 368 }
 369
 370 static __always_inline bool
 371 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
 372 {
 373         size_t i;
 374
 375         for (i = 0; i < size; i++) {
 376                 if (!kasan_shadow_1byte_isvalid(addr+i, code))
 377                         return (false);
 378         }
 379
 380         return (true);
 381 }
 382
 383 static __always_inline void
 384 kasan_shadow_check(unsigned long addr, size_t size, bool write,
 385     unsigned long retaddr)
 386 {
 387         uint8_t code;
 388         bool valid;
 389
 390         if (__predict_false(!kasan_enabled))
 391                 return;
 392         if (__predict_false(size == 0))
 393                 return;
 394         if (__predict_false(kasan_md_unsupported(addr)))
 395                 return;
 396         if (KERNEL_PANICKED())
 397                 return;
 398
 399         if (__builtin_constant_p(size)) {
 400                 switch (size) {
 401                 case 1:
 402                         valid = kasan_shadow_1byte_isvalid(addr, &code);
 403                         break;
 404                 case 2:
 405                         valid = kasan_shadow_2byte_isvalid(addr, &code);
 406                         break;
 407                 case 4:
 408                         valid = kasan_shadow_4byte_isvalid(addr, &code);
 409                         break;
 410                 case 8:
 411                         valid = kasan_shadow_8byte_isvalid(addr, &code);
 412                         break;
 413                 default:
 414                         valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 415                         break;
 416                 }
 417         } else {
 418                 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 419         }
 420
 421         if (__predict_false(!valid)) {
 422                 kasan_report(addr, size, write, retaddr, code);
 423         }
 424 }
 425
 426 /* -------------------------------------------------------------------------- */
 427
 428 void *
 429 kasan_memcpy(void *dst, const void *src, size_t len)
 430 {
 431         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 432         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 433         return (__builtin_memcpy(dst, src, len));
 434 }
 435
 436 int
 437 kasan_memcmp(const void *b1, const void *b2, size_t len)
 438 {
 439         kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
 440         kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
 441         return (__builtin_memcmp(b1, b2, len));
 442 }
 443
 444 void *
 445 kasan_memset(void *b, int c, size_t len)
 446 {
 447         kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
 448         return (__builtin_memset(b, c, len));
 449 }
 450
 451 void *
 452 kasan_memmove(void *dst, const void *src, size_t len)
 453 {
 454         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 455         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 456         return (__builtin_memmove(dst, src, len));
 457 }
 458
 459 size_t
 460 kasan_strlen(const char *str)
 461 {
 462         const char *s;
 463
 464         s = str;
 465         while (1) {
 466                 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
 467                 if (*s == '\0')
 468                         break;
 469                 s++;
 470         }
 471
 472         return (s - str);
 473 }
 474
 475 char *
 476 kasan_strcpy(char *dst, const char *src)
 477 {
 478         char *save = dst;
 479
 480         while (1) {
 481                 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
 482                 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
 483                 *dst = *src;
 484                 if (*src == '\0')
 485                         break;
 486                 src++, dst++;
 487         }
 488
 489         return save;
 490 }
 491
 492 int
 493 kasan_strcmp(const char *s1, const char *s2)
 494 {
 495         while (1) {
 496                 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
 497                 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
 498                 if (*s1 != *s2)
 499                         break;
 500                 if (*s1 == '\0')
 501                         return 0;
 502                 s1++, s2++;
 503         }
 504
 505         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 506 }
 507
 508 int
 509 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
 510 {
 511         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 512         return (copyin(uaddr, kaddr, len));
 513 }
 514
 515 int
 516 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 517 {
 518         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 519         return (copyinstr(uaddr, kaddr, len, done));
 520 }
 521
 522 int
 523 kasan_copyout(const void *kaddr, void *uaddr, size_t len)
 524 {
 525         kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
 526         return (copyout(kaddr, uaddr, len));
 527 }
 528
 529 /* -------------------------------------------------------------------------- */
 530
 531 #include <machine/atomic.h>
 532 #include <sys/atomic_san.h>
 533
 534 #define _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 535         void kasan_atomic_add_##name(volatile type *ptr, type val)      \
 536         {                                                               \
 537                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 538                     __RET_ADDR);                                        \
 539                 atomic_add_##name(ptr, val);                            \
 540         }
 541
 542 #define ASAN_ATOMIC_FUNC_ADD(name, type)                                \
 543         _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 544         _ASAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
 545         _ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
 546
 547 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 548         void kasan_atomic_subtract_##name(volatile type *ptr, type val) \
 549         {                                                               \
 550                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 551                     __RET_ADDR);                                        \
 552                 atomic_subtract_##name(ptr, val);                       \
 553         }
 554
 555 #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
 556         _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 557         _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
 558         _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
 559
 560 #define _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 561         void kasan_atomic_set_##name(volatile type *ptr, type val)      \
 562         {                                                               \
 563                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 564                     __RET_ADDR);                                        \
 565                 atomic_set_##name(ptr, val);                            \
 566         }
 567
 568 #define ASAN_ATOMIC_FUNC_SET(name, type)                                \
 569         _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 570         _ASAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
 571         _ASAN_ATOMIC_FUNC_SET(rel_##name, type)
 572
 573 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 574         void kasan_atomic_clear_##name(volatile type *ptr, type val)    \
 575         {                                                               \
 576                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 577                     __RET_ADDR);                                        \
 578                 atomic_clear_##name(ptr, val);                          \
 579         }
 580
 581 #define ASAN_ATOMIC_FUNC_CLEAR(name, type)                              \
 582         _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 583         _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
 584         _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
 585
 586 #define ASAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
 587         type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \
 588         {                                                               \
 589                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 590                     __RET_ADDR);                                        \
 591                 return (atomic_fetchadd_##name(ptr, val));              \
 592         }
 593
 594 #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
 595         type kasan_atomic_readandclear_##name(volatile type *ptr)       \
 596         {                                                               \
 597                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 598                     __RET_ADDR);                                        \
 599                 return (atomic_readandclear_##name(ptr));               \
 600         }
 601
 602 #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
 603         int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
 604         {                                                               \
 605                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 606                     __RET_ADDR);                                        \
 607                 return (atomic_testandclear_##name(ptr, v));            \
 608         }
 609
 610 #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
 611         int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
 612         {                                                               \
 613                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 614                     __RET_ADDR);                                        \
 615                 return (atomic_testandset_##name(ptr, v));              \
 616         }
 617
 618 #define ASAN_ATOMIC_FUNC_SWAP(name, type)                               \
 619         type kasan_atomic_swap_##name(volatile type *ptr, type val)     \
 620         {                                                               \
 621                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 622                     __RET_ADDR);                                        \
 623                 return (atomic_swap_##name(ptr, val));                  \
 624         }
 625
 626 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 627         int kasan_atomic_cmpset_##name(volatile type *ptr, type oval,   \
 628             type nval)                                                  \
 629         {                                                               \
 630                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 631                     __RET_ADDR);                                        \
 632                 return (atomic_cmpset_##name(ptr, oval, nval));         \
 633         }
 634
 635 #define ASAN_ATOMIC_FUNC_CMPSET(name, type)                             \
 636         _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 637         _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
 638         _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
 639
 640 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 641         int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
 642             type nval)                                                  \
 643         {                                                               \
 644                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 645                     __RET_ADDR);                                        \
 646                 return (atomic_fcmpset_##name(ptr, oval, nval));        \
 647         }
 648
 649 #define ASAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
 650         _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 651         _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
 652         _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
 653
 654 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
 655         void kasan_atomic_thread_fence_##name(void)                     \
 656         {                                                               \
 657                 atomic_thread_fence_##name();                           \
 658         }
 659
 660 #define _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 661         type kasan_atomic_load_##name(volatile type *ptr)               \
 662         {                                                               \
 663                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 664                     __RET_ADDR);                                        \
 665                 return (atomic_load_##name(ptr));                       \
 666         }
 667
 668 #define ASAN_ATOMIC_FUNC_LOAD(name, type)                               \
 669         _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 670         _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
 671
 672 #define _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 673         void kasan_atomic_store_##name(volatile type *ptr, type val)    \
 674         {                                                               \
 675                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 676                     __RET_ADDR);                                        \
 677                 atomic_store_##name(ptr, val);                          \
 678         }
 679
 680 #define ASAN_ATOMIC_FUNC_STORE(name, type)                              \
 681         _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 682         _ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
 683
 684 ASAN_ATOMIC_FUNC_ADD(8, uint8_t);
 685 ASAN_ATOMIC_FUNC_ADD(16, uint16_t);
 686 ASAN_ATOMIC_FUNC_ADD(32, uint32_t);
 687 ASAN_ATOMIC_FUNC_ADD(64, uint64_t);
 688 ASAN_ATOMIC_FUNC_ADD(int, u_int);
 689 ASAN_ATOMIC_FUNC_ADD(long, u_long);
 690 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
 691
 692 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
 693 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
 694 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
 695 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
 696 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
 697 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
 698 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
 699
 700 ASAN_ATOMIC_FUNC_SET(8, uint8_t);
 701 ASAN_ATOMIC_FUNC_SET(16, uint16_t);
 702 ASAN_ATOMIC_FUNC_SET(32, uint32_t);
 703 ASAN_ATOMIC_FUNC_SET(64, uint64_t);
 704 ASAN_ATOMIC_FUNC_SET(int, u_int);
 705 ASAN_ATOMIC_FUNC_SET(long, u_long);
 706 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
 707
 708 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
 709 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
 710 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
 711 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
 712 ASAN_ATOMIC_FUNC_CLEAR(int, u_int);
 713 ASAN_ATOMIC_FUNC_CLEAR(long, u_long);
 714 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
 715
 716 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
 717 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
 718 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int);
 719 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long);
 720
 721 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
 722 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
 723 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
 724 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
 725 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
 726
 727 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
 728 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
 729 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
 730 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
 731
 732 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
 733 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
 734 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
 735 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
 736
 737 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t);
 738 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t);
 739 ASAN_ATOMIC_FUNC_SWAP(int, u_int);
 740 ASAN_ATOMIC_FUNC_SWAP(long, u_long);
 741 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
 742
 743 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
 744 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
 745 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
 746 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
 747 ASAN_ATOMIC_FUNC_CMPSET(int, u_int);
 748 ASAN_ATOMIC_FUNC_CMPSET(long, u_long);
 749 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
 750
 751 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
 752 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
 753 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
 754 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
 755 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int);
 756 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long);
 757 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
 758
 759 _ASAN_ATOMIC_FUNC_LOAD(bool, bool);
 760 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t);
 761 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t);
 762 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t);
 763 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t);
 764 ASAN_ATOMIC_FUNC_LOAD(char, u_char);
 765 ASAN_ATOMIC_FUNC_LOAD(short, u_short);
 766 ASAN_ATOMIC_FUNC_LOAD(int, u_int);
 767 ASAN_ATOMIC_FUNC_LOAD(long, u_long);
 768 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
 769
 770 _ASAN_ATOMIC_FUNC_STORE(bool, bool);
 771 ASAN_ATOMIC_FUNC_STORE(8, uint8_t);
 772 ASAN_ATOMIC_FUNC_STORE(16, uint16_t);
 773 ASAN_ATOMIC_FUNC_STORE(32, uint32_t);
 774 ASAN_ATOMIC_FUNC_STORE(64, uint64_t);
 775 ASAN_ATOMIC_FUNC_STORE(char, u_char);
 776 ASAN_ATOMIC_FUNC_STORE(short, u_short);
 777 ASAN_ATOMIC_FUNC_STORE(int, u_int);
 778 ASAN_ATOMIC_FUNC_STORE(long, u_long);
 779 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
 780
 781 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq);
 782 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel);
 783 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
 784 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
 785
 786 void
 787 kasan_atomic_interrupt_fence(void)
 788 {
 789 }
 790
 791 /* -------------------------------------------------------------------------- */
 792
 793 #include <sys/bus.h>
 794 #include <machine/bus.h>
 795 #include <sys/bus_san.h>
 796
 797 int
 798 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
 799     int flags, bus_space_handle_t *handlep)
 800 {
 801         return (bus_space_map(tag, hnd, size, flags, handlep));
 802 }
 803
 804 void
 805 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
 806     bus_size_t size)
 807 {
 808         bus_space_unmap(tag, hnd, size);
 809 }
 810
 811 int
 812 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
 813     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
 814 {
 815         return (bus_space_subregion(tag, hnd, offset, size, handlep));
 816 }
 817
 818 void
 819 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
 820     bus_size_t size)
 821 {
 822         bus_space_free(tag, hnd, size);
 823 }
 824
 825 void
 826 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
 827     bus_size_t offset, bus_size_t size, int flags)
 828 {
 829         bus_space_barrier(tag, hnd, offset, size, flags);
 830 }
 831
 832 #define ASAN_BUS_READ_FUNC(func, width, type)                           \
 833         type kasan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
 834             bus_space_handle_t hnd, bus_size_t offset)                  \
 835         {                                                               \
 836                 return (bus_space_read##func##_##width(tag, hnd,        \
 837                     offset));                                           \
 838         }                                                               \
 839
 840 #define ASAN_BUS_READ_PTR_FUNC(func, width, type)                       \
 841         void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
 842             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
 843             bus_size_t count)                                           \
 844         {                                                               \
 845                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 846                     false, __RET_ADDR);                                 \
 847                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
 848                     count);                                             \
 849         }
 850
 851 ASAN_BUS_READ_FUNC(, 1, uint8_t)
 852 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t)
 853 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
 854 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
 855 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
 856 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
 857
 858 ASAN_BUS_READ_FUNC(, 2, uint16_t)
 859 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t)
 860 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
 861 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
 862 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
 863 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
 864
 865 ASAN_BUS_READ_FUNC(, 4, uint32_t)
 866 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t)
 867 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
 868 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
 869 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
 870 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
 871
 872 ASAN_BUS_READ_FUNC(, 8, uint64_t)
 873 #if defined(__aarch64__)
 874 ASAN_BUS_READ_FUNC(_stream, 8, uint64_t)
 875 ASAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
 876 ASAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
 877 ASAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
 878 ASAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
 879 #endif
 880
 881 #define ASAN_BUS_WRITE_FUNC(func, width, type)                          \
 882         void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \
 883             bus_space_handle_t hnd, bus_size_t offset, type value)      \
 884         {                                                               \
 885                 bus_space_write##func##_##width(tag, hnd, offset, value);\
 886         }                                                               \
 887
 888 #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
 889         void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
 890             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
 891             bus_size_t count)                                           \
 892         {                                                               \
 893                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 894                     true, __RET_ADDR);                                  \
 895                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
 896                     count);                                             \
 897         }
 898
 899 ASAN_BUS_WRITE_FUNC(, 1, uint8_t)
 900 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
 901 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
 902 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
 903 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
 904 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
 905
 906 ASAN_BUS_WRITE_FUNC(, 2, uint16_t)
 907 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
 908 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
 909 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
 910 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
 911 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
 912
 913 ASAN_BUS_WRITE_FUNC(, 4, uint32_t)
 914 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
 915 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
 916 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
 917 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
 918 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
 919
 920 ASAN_BUS_WRITE_FUNC(, 8, uint64_t)
 921
 922 #define ASAN_BUS_SET_FUNC(func, width, type)                            \
 923         void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
 924             bus_space_handle_t hnd, bus_size_t offset, type value,      \
 925             bus_size_t count)                                           \
 926         {                                                               \
 927                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
 928                     count);                                             \
 929         }
 930
 931 ASAN_BUS_SET_FUNC(multi, 1, uint8_t)
 932 ASAN_BUS_SET_FUNC(region, 1, uint8_t)
 933 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
 934 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
 935
 936 ASAN_BUS_SET_FUNC(multi, 2, uint16_t)
 937 ASAN_BUS_SET_FUNC(region, 2, uint16_t)
 938 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
 939 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
 940
 941 ASAN_BUS_SET_FUNC(multi, 4, uint32_t)
 942 ASAN_BUS_SET_FUNC(region, 4, uint32_t)
 943 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
 944 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
 945
 946 #define ASAN_BUS_PEEK_FUNC(width, type)                                 \
 947         int kasan_bus_space_peek_##width(bus_space_tag_t tag,           \
 948             bus_space_handle_t hnd, bus_size_t offset, type *valuep)    \
 949         {                                                               \
 950                 return (bus_space_peek_##width(tag, hnd, offset,        \
 951                     valuep));                                           \
 952         }
 953
 954 ASAN_BUS_PEEK_FUNC(1, uint8_t)
 955 ASAN_BUS_PEEK_FUNC(2, uint16_t)
 956 ASAN_BUS_PEEK_FUNC(4, uint32_t)
 957 ASAN_BUS_PEEK_FUNC(8, uint64_t)
 958
 959 #define ASAN_BUS_POKE_FUNC(width, type)                                 \
 960         int kasan_bus_space_poke_##width(bus_space_tag_t tag,           \
 961             bus_space_handle_t hnd, bus_size_t offset, type value)      \
 962         {                                                               \
 963                 return (bus_space_poke_##width(tag, hnd, offset,        \
 964                     value));                                            \
 965         }
 966
 967 ASAN_BUS_POKE_FUNC(1, uint8_t)
 968 ASAN_BUS_POKE_FUNC(2, uint16_t)
 969 ASAN_BUS_POKE_FUNC(4, uint32_t)
 970 ASAN_BUS_POKE_FUNC(8, uint64_t)
 971
 972 /* -------------------------------------------------------------------------- */
 973
 974 void __asan_register_globals(struct __asan_global *, size_t);
 975 void __asan_unregister_globals(struct __asan_global *, size_t);
 976
 977 void
 978 __asan_register_globals(struct __asan_global *globals, size_t n)
 979 {
 980         size_t i;
 981
 982         for (i = 0; i < n; i++) {
 983                 kasan_mark(globals[i].beg, globals[i].size,
 984                     globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
 985         }
 986 }
 987
 988 void
 989 __asan_unregister_globals(struct __asan_global *globals, size_t n)
 990 {
 991         size_t i;
 992
 993         for (i = 0; i < n; i++) {
 994                 kasan_mark(globals[i].beg, globals[i].size_with_redzone,
 995                     globals[i].size_with_redzone, 0);
 996         }
 997 }
 998
 999 #define ASAN_LOAD_STORE(size)                                   \
1000         void __asan_load##size(unsigned long);                  \
1001         void __asan_load##size(unsigned long addr)              \
1002         {                                                       \
1003                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1004         }                                                       \
1005         void __asan_load##size##_noabort(unsigned long);        \
1006         void __asan_load##size##_noabort(unsigned long addr)    \
1007         {                                                       \
1008                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1009         }                                                       \
1010         void __asan_store##size(unsigned long);                 \
1011         void __asan_store##size(unsigned long addr)             \
1012         {                                                       \
1013                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1014         }                                                       \
1015         void __asan_store##size##_noabort(unsigned long);       \
1016         void __asan_store##size##_noabort(unsigned long addr)   \
1017         {                                                       \
1018                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1019         }
1020
1021 ASAN_LOAD_STORE(1);
1022 ASAN_LOAD_STORE(2);
1023 ASAN_LOAD_STORE(4);
1024 ASAN_LOAD_STORE(8);
1025 ASAN_LOAD_STORE(16);
1026
1027 void __asan_loadN(unsigned long, size_t);
1028 void __asan_loadN_noabort(unsigned long, size_t);
1029 void __asan_storeN(unsigned long, size_t);
1030 void __asan_storeN_noabort(unsigned long, size_t);
1031 void __asan_handle_no_return(void);
1032
1033 void
1034 __asan_loadN(unsigned long addr, size_t size)
1035 {
1036         kasan_shadow_check(addr, size, false, __RET_ADDR);
1037 }
1038
1039 void
1040 __asan_loadN_noabort(unsigned long addr, size_t size)
1041 {
1042         kasan_shadow_check(addr, size, false, __RET_ADDR);
1043 }
1044
1045 void
1046 __asan_storeN(unsigned long addr, size_t size)
1047 {
1048         kasan_shadow_check(addr, size, true, __RET_ADDR);
1049 }
1050
1051 void
1052 __asan_storeN_noabort(unsigned long addr, size_t size)
1053 {
1054         kasan_shadow_check(addr, size, true, __RET_ADDR);
1055 }
1056
1057 void
1058 __asan_handle_no_return(void)
1059 {
1060         /* nothing */
1061 }
1062
1063 #define ASAN_SET_SHADOW(byte) \
1064         void __asan_set_shadow_##byte(void *, size_t);                  \
1065         void __asan_set_shadow_##byte(void *addr, size_t size)          \
1066         {                                                               \
1067                 __builtin_memset((void *)addr, 0x##byte, size);         \
1068         }
1069
1070 ASAN_SET_SHADOW(00);
1071 ASAN_SET_SHADOW(f1);
1072 ASAN_SET_SHADOW(f2);
1073 ASAN_SET_SHADOW(f3);
1074 ASAN_SET_SHADOW(f5);
1075 ASAN_SET_SHADOW(f8);
1076
1077 void __asan_poison_stack_memory(const void *, size_t);
1078 void __asan_unpoison_stack_memory(const void *, size_t);
1079
1080 void
1081 __asan_poison_stack_memory(const void *addr, size_t size)
1082 {
1083         size = roundup(size, KASAN_SHADOW_SCALE);
1084         kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1085 }
1086
1087 void
1088 __asan_unpoison_stack_memory(const void *addr, size_t size)
1089 {
1090         kasan_shadow_Nbyte_markvalid(addr, size);
1091 }
1092
1093 void __asan_alloca_poison(const void *, size_t);
1094 void __asan_allocas_unpoison(const void *, const void *);
1095
1096 void
1097 __asan_alloca_poison(const void *addr, size_t size)
1098 {
1099         const void *l, *r;
1100
1101         KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0,
1102             ("%s: invalid address %p", __func__, addr));
1103
1104         l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1105         r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1106
1107         kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1108         kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1109             KASAN_STACK_MID);
1110         kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1111 }
1112
1113 void
1114 __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1115 {
1116         size_t size;
1117
1118         if (__predict_false(!stkbegin))
1119                 return;
1120         if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1121                 return;
1122         size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1123
1124         kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1125 }
1126
1127 void __asan_poison_memory_region(const void *addr, size_t size);
1128 void __asan_unpoison_memory_region(const void *addr, size_t size);
1129
1130 void
1131 __asan_poison_memory_region(const void *addr, size_t size)
1132 {
1133 }
1134
1135 void
1136 __asan_unpoison_memory_region(const void *addr, size_t size)
1137 {
1138 }