sys/kern/subr_asan.c

   1 /*      $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $      */
   2
   3 /*
   4  * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
   5  * All rights reserved.
   6  *
   7  * This code is part of the KASAN subsystem of the NetBSD kernel.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  23  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  25  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  26  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 #define SAN_RUNTIME
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35 #if 0
  36 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $");
  37 #endif
  38
  39 #include <sys/param.h>
  40 #include <sys/systm.h>
  41 #include <sys/asan.h>
  42 #include <sys/kernel.h>
  43 #include <sys/stack.h>
  44 #include <sys/sysctl.h>
  45
  46 #include <machine/asan.h>
  47
  48 /* ASAN constants. Part of the compiler ABI. */
  49 #define KASAN_SHADOW_MASK               (KASAN_SHADOW_SCALE - 1)
  50 #define KASAN_ALLOCA_SCALE_SIZE         32
  51
  52 /* ASAN ABI version. */
  53 #if defined(__clang__) && (__clang_major__ - 0 >= 6)
  54 #define ASAN_ABI_VERSION        8
  55 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
  56 #define ASAN_ABI_VERSION        8
  57 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
  58 #define ASAN_ABI_VERSION        6
  59 #else
  60 #error "Unsupported compiler version"
  61 #endif
  62
  63 #define __RET_ADDR      (unsigned long)__builtin_return_address(0)
  64
  65 /* Global variable descriptor. Part of the compiler ABI.  */
  66 struct __asan_global_source_location {
  67         const char *filename;
  68         int line_no;
  69         int column_no;
  70 };
  71
  72 struct __asan_global {
  73         const void *beg;                /* address of the global variable */
  74         size_t size;                    /* size of the global variable */
  75         size_t size_with_redzone;       /* size with the redzone */
  76         const void *name;               /* name of the variable */
  77         const void *module_name;        /* name of the module where the var is declared */
  78         unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
  79         struct __asan_global_source_location *location;
  80 #if ASAN_ABI_VERSION >= 7
  81         uintptr_t odr_indicator;        /* the address of the ODR indicator symbol */
  82 #endif
  83 };
  84
  85 FEATURE(kasan, "Kernel address sanitizer");
  86
  87 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  88     "KASAN options");
  89
  90 static int panic_on_violation = 1;
  91 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN,
  92     &panic_on_violation, 0,
  93     "Panic if an invalid access is detected");
  94
  95 static bool kasan_enabled __read_mostly = false;
  96
  97 /* -------------------------------------------------------------------------- */
  98
  99 void
 100 kasan_shadow_map(vm_offset_t addr, size_t size)
 101 {
 102         size_t sz, npages, i;
 103         vm_offset_t sva, eva;
 104
 105         KASSERT(addr % KASAN_SHADOW_SCALE == 0,
 106             ("%s: invalid address %#lx", __func__, addr));
 107
 108         sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE;
 109
 110         sva = kasan_md_addr_to_shad(addr);
 111         eva = kasan_md_addr_to_shad(addr) + sz;
 112
 113         sva = rounddown(sva, PAGE_SIZE);
 114         eva = roundup(eva, PAGE_SIZE);
 115
 116         npages = (eva - sva) / PAGE_SIZE;
 117
 118         KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS,
 119             ("%s: invalid address range %#lx-%#lx", __func__, sva, eva));
 120
 121         for (i = 0; i < npages; i++)
 122                 pmap_san_enter(sva + ptoa(i));
 123 }
 124
 125 void
 126 kasan_init(void)
 127 {
 128         int disabled;
 129
 130         disabled = 0;
 131         TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled);
 132         if (disabled)
 133                 return;
 134
 135         /* MD initialization. */
 136         kasan_md_init();
 137
 138         /* Now officially enabled. */
 139         kasan_enabled = true;
 140 }
 141
 142 static inline const char *
 143 kasan_code_name(uint8_t code)
 144 {
 145         switch (code) {
 146         case KASAN_GENERIC_REDZONE:
 147                 return "GenericRedZone";
 148         case KASAN_MALLOC_REDZONE:
 149                 return "MallocRedZone";
 150         case KASAN_KMEM_REDZONE:
 151                 return "KmemRedZone";
 152         case KASAN_UMA_FREED:
 153                 return "UMAUseAfterFree";
 154         case KASAN_KSTACK_FREED:
 155                 return "KernelStack";
 156         case KASAN_EXEC_ARGS_FREED:
 157                 return "ExecKVA";
 158         case 1 ... 7:
 159                 return "RedZonePartial";
 160         case KASAN_STACK_LEFT:
 161                 return "StackLeft";
 162         case KASAN_STACK_MID:
 163                 return "StackMiddle";
 164         case KASAN_STACK_RIGHT:
 165                 return "StackRight";
 166         case KASAN_USE_AFTER_RET:
 167                 return "UseAfterRet";
 168         case KASAN_USE_AFTER_SCOPE:
 169                 return "UseAfterScope";
 170         default:
 171                 return "Unknown";
 172         }
 173 }
 174
 175 #define REPORT(f, ...) do {                             \
 176         if (panic_on_violation) {                       \
 177                 kasan_enabled = false;                  \
 178                 panic(f, __VA_ARGS__);                  \
 179         } else {                                        \
 180                 struct stack st;                        \
 181                                                         \
 182                 stack_save(&st);                        \
 183                 printf(f "\n", __VA_ARGS__);            \
 184                 stack_print_ddb(&st);                   \
 185         }                                               \
 186 } while (0)
 187
 188 static void
 189 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
 190     uint8_t code)
 191 {
 192         REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)",
 193             size, (write ? "write" : "read"), addr, kasan_code_name(code),
 194             code);
 195 }
 196
 197 static __always_inline void
 198 kasan_shadow_1byte_markvalid(unsigned long addr)
 199 {
 200         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 201         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 202
 203         *byte = last;
 204 }
 205
 206 static __always_inline void
 207 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
 208 {
 209         size_t i;
 210
 211         for (i = 0; i < size; i++) {
 212                 kasan_shadow_1byte_markvalid((unsigned long)addr + i);
 213         }
 214 }
 215
 216 static __always_inline void
 217 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
 218 {
 219         void *shad;
 220
 221         if (__predict_false(size == 0))
 222                 return;
 223         if (__predict_false(kasan_md_unsupported((vm_offset_t)addr)))
 224                 return;
 225
 226         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 227             ("%s: invalid address %p", __func__, addr));
 228         KASSERT(size % KASAN_SHADOW_SCALE == 0,
 229             ("%s: invalid size %zu", __func__, size));
 230
 231         shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr);
 232         size = size >> KASAN_SHADOW_SCALE_SHIFT;
 233
 234         __builtin_memset(shad, code, size);
 235 }
 236
 237 /*
 238  * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
 239  * and the rest as invalid. There are generally two use cases:
 240  *
 241  *  o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
 242  *    the redzone at the end of the buffer as invalid. If the entire is to be
 243  *    marked invalid, origsize will be 0.
 244  *
 245  *  o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
 246  */
 247 void
 248 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
 249 {
 250         size_t i, n, redz;
 251         int8_t *shad;
 252
 253         if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS &&
 254             (vm_offset_t)addr < DMAP_MAX_ADDRESS)
 255                 return;
 256
 257         KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS &&
 258             (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS,
 259             ("%s: invalid address %p", __func__, addr));
 260         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 261             ("%s: invalid address %p", __func__, addr));
 262         redz = redzsize - roundup(size, KASAN_SHADOW_SCALE);
 263         KASSERT(redz % KASAN_SHADOW_SCALE == 0,
 264             ("%s: invalid size %zu", __func__, redz));
 265         shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr);
 266
 267         /* Chunks of 8 bytes, valid. */
 268         n = size / KASAN_SHADOW_SCALE;
 269         for (i = 0; i < n; i++) {
 270                 *shad++ = 0;
 271         }
 272
 273         /* Possibly one chunk, mid. */
 274         if ((size & KASAN_SHADOW_MASK) != 0) {
 275                 *shad++ = (size & KASAN_SHADOW_MASK);
 276         }
 277
 278         /* Chunks of 8 bytes, invalid. */
 279         n = redz / KASAN_SHADOW_SCALE;
 280         for (i = 0; i < n; i++) {
 281                 *shad++ = code;
 282         }
 283 }
 284
 285 /* -------------------------------------------------------------------------- */
 286
 287 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size)                 \
 288         (addr >> KASAN_SHADOW_SCALE_SHIFT) !=                   \
 289             ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
 290
 291 static __always_inline bool
 292 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
 293 {
 294         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 295         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 296
 297         if (__predict_true(*byte == 0 || last <= *byte)) {
 298                 return (true);
 299         }
 300         *code = *byte;
 301         return (false);
 302 }
 303
 304 static __always_inline bool
 305 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
 306 {
 307         int8_t *byte, last;
 308
 309         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
 310                 return (kasan_shadow_1byte_isvalid(addr, code) &&
 311                     kasan_shadow_1byte_isvalid(addr+1, code));
 312         }
 313
 314         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 315         last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
 316
 317         if (__predict_true(*byte == 0 || last <= *byte)) {
 318                 return (true);
 319         }
 320         *code = *byte;
 321         return (false);
 322 }
 323
 324 static __always_inline bool
 325 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
 326 {
 327         int8_t *byte, last;
 328
 329         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
 330                 return (kasan_shadow_2byte_isvalid(addr, code) &&
 331                     kasan_shadow_2byte_isvalid(addr+2, code));
 332         }
 333
 334         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 335         last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
 336
 337         if (__predict_true(*byte == 0 || last <= *byte)) {
 338                 return (true);
 339         }
 340         *code = *byte;
 341         return (false);
 342 }
 343
 344 static __always_inline bool
 345 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
 346 {
 347         int8_t *byte, last;
 348
 349         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
 350                 return (kasan_shadow_4byte_isvalid(addr, code) &&
 351                     kasan_shadow_4byte_isvalid(addr+4, code));
 352         }
 353
 354         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 355         last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
 356
 357         if (__predict_true(*byte == 0 || last <= *byte)) {
 358                 return (true);
 359         }
 360         *code = *byte;
 361         return (false);
 362 }
 363
 364 static __always_inline bool
 365 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
 366 {
 367         size_t i;
 368
 369         for (i = 0; i < size; i++) {
 370                 if (!kasan_shadow_1byte_isvalid(addr+i, code))
 371                         return (false);
 372         }
 373
 374         return (true);
 375 }
 376
 377 static __always_inline void
 378 kasan_shadow_check(unsigned long addr, size_t size, bool write,
 379     unsigned long retaddr)
 380 {
 381         uint8_t code;
 382         bool valid;
 383
 384         if (__predict_false(!kasan_enabled))
 385                 return;
 386         if (__predict_false(size == 0))
 387                 return;
 388         if (__predict_false(kasan_md_unsupported(addr)))
 389                 return;
 390         if (__predict_false(panicstr != NULL))
 391                 return;
 392
 393         if (__builtin_constant_p(size)) {
 394                 switch (size) {
 395                 case 1:
 396                         valid = kasan_shadow_1byte_isvalid(addr, &code);
 397                         break;
 398                 case 2:
 399                         valid = kasan_shadow_2byte_isvalid(addr, &code);
 400                         break;
 401                 case 4:
 402                         valid = kasan_shadow_4byte_isvalid(addr, &code);
 403                         break;
 404                 case 8:
 405                         valid = kasan_shadow_8byte_isvalid(addr, &code);
 406                         break;
 407                 default:
 408                         valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 409                         break;
 410                 }
 411         } else {
 412                 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 413         }
 414
 415         if (__predict_false(!valid)) {
 416                 kasan_report(addr, size, write, retaddr, code);
 417         }
 418 }
 419
 420 /* -------------------------------------------------------------------------- */
 421
 422 void *
 423 kasan_memcpy(void *dst, const void *src, size_t len)
 424 {
 425         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 426         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 427         return (__builtin_memcpy(dst, src, len));
 428 }
 429
 430 int
 431 kasan_memcmp(const void *b1, const void *b2, size_t len)
 432 {
 433         kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
 434         kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
 435         return (__builtin_memcmp(b1, b2, len));
 436 }
 437
 438 void *
 439 kasan_memset(void *b, int c, size_t len)
 440 {
 441         kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
 442         return (__builtin_memset(b, c, len));
 443 }
 444
 445 void *
 446 kasan_memmove(void *dst, const void *src, size_t len)
 447 {
 448         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 449         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 450         return (__builtin_memmove(dst, src, len));
 451 }
 452
 453 size_t
 454 kasan_strlen(const char *str)
 455 {
 456         const char *s;
 457
 458         s = str;
 459         while (1) {
 460                 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
 461                 if (*s == '\0')
 462                         break;
 463                 s++;
 464         }
 465
 466         return (s - str);
 467 }
 468
 469 char *
 470 kasan_strcpy(char *dst, const char *src)
 471 {
 472         char *save = dst;
 473
 474         while (1) {
 475                 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
 476                 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
 477                 *dst = *src;
 478                 if (*src == '\0')
 479                         break;
 480                 src++, dst++;
 481         }
 482
 483         return save;
 484 }
 485
 486 int
 487 kasan_strcmp(const char *s1, const char *s2)
 488 {
 489         while (1) {
 490                 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
 491                 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
 492                 if (*s1 != *s2)
 493                         break;
 494                 if (*s1 == '\0')
 495                         return 0;
 496                 s1++, s2++;
 497         }
 498
 499         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 500 }
 501
 502 int
 503 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
 504 {
 505         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 506         return (copyin(uaddr, kaddr, len));
 507 }
 508
 509 int
 510 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 511 {
 512         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 513         return (copyinstr(uaddr, kaddr, len, done));
 514 }
 515
 516 int
 517 kasan_copyout(const void *kaddr, void *uaddr, size_t len)
 518 {
 519         kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
 520         return (copyout(kaddr, uaddr, len));
 521 }
 522
 523 /* -------------------------------------------------------------------------- */
 524
 525 int
 526 kasan_fubyte(volatile const void *base)
 527 {
 528         return (fubyte(base));
 529 }
 530
 531 int
 532 kasan_fuword16(volatile const void *base)
 533 {
 534         return (fuword16(base));
 535 }
 536
 537 int
 538 kasan_fueword(volatile const void *base, long *val)
 539 {
 540         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 541         return (fueword(base, val));
 542 }
 543
 544 int
 545 kasan_fueword32(volatile const void *base, int32_t *val)
 546 {
 547         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 548         return (fueword32(base, val));
 549 }
 550
 551 int
 552 kasan_fueword64(volatile const void *base, int64_t *val)
 553 {
 554         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 555         return (fueword64(base, val));
 556 }
 557
 558 int
 559 kasan_subyte(volatile void *base, int byte)
 560 {
 561         return (subyte(base, byte));
 562 }
 563
 564 int
 565 kasan_suword(volatile void *base, long word)
 566 {
 567         return (suword(base, word));
 568 }
 569
 570 int
 571 kasan_suword16(volatile void *base, int word)
 572 {
 573         return (suword16(base, word));
 574 }
 575
 576 int
 577 kasan_suword32(volatile void *base, int32_t word)
 578 {
 579         return (suword32(base, word));
 580 }
 581
 582 int
 583 kasan_suword64(volatile void *base, int64_t word)
 584 {
 585         return (suword64(base, word));
 586 }
 587
 588 int
 589 kasan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
 590     uint32_t newval)
 591 {
 592         kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
 593             __RET_ADDR);
 594         return (casueword32(base, oldval, oldvalp, newval));
 595 }
 596
 597 int
 598 kasan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
 599     u_long newval)
 600 {
 601         kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
 602             __RET_ADDR);
 603         return (casueword(base, oldval, oldvalp, newval));
 604 }
 605
 606 /* -------------------------------------------------------------------------- */
 607
 608 #include <machine/atomic.h>
 609 #include <sys/atomic_san.h>
 610
 611 #define _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 612         void kasan_atomic_add_##name(volatile type *ptr, type val)      \
 613         {                                                               \
 614                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 615                     __RET_ADDR);                                        \
 616                 atomic_add_##name(ptr, val);                            \
 617         }
 618
 619 #define ASAN_ATOMIC_FUNC_ADD(name, type)                                \
 620         _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 621         _ASAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
 622         _ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
 623
 624 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 625         void kasan_atomic_subtract_##name(volatile type *ptr, type val) \
 626         {                                                               \
 627                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 628                     __RET_ADDR);                                        \
 629                 atomic_subtract_##name(ptr, val);                       \
 630         }
 631
 632 #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
 633         _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 634         _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
 635         _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
 636
 637 #define _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 638         void kasan_atomic_set_##name(volatile type *ptr, type val)      \
 639         {                                                               \
 640                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 641                     __RET_ADDR);                                        \
 642                 atomic_set_##name(ptr, val);                            \
 643         }
 644
 645 #define ASAN_ATOMIC_FUNC_SET(name, type)                                \
 646         _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 647         _ASAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
 648         _ASAN_ATOMIC_FUNC_SET(rel_##name, type)
 649
 650 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 651         void kasan_atomic_clear_##name(volatile type *ptr, type val)    \
 652         {                                                               \
 653                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 654                     __RET_ADDR);                                        \
 655                 atomic_clear_##name(ptr, val);                          \
 656         }
 657
 658 #define ASAN_ATOMIC_FUNC_CLEAR(name, type)                              \
 659         _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 660         _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
 661         _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
 662
 663 #define ASAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
 664         type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \
 665         {                                                               \
 666                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 667                     __RET_ADDR);                                        \
 668                 return (atomic_fetchadd_##name(ptr, val));              \
 669         }
 670
 671 #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
 672         type kasan_atomic_readandclear_##name(volatile type *ptr)       \
 673         {                                                               \
 674                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 675                     __RET_ADDR);                                        \
 676                 return (atomic_readandclear_##name(ptr));               \
 677         }
 678
 679 #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
 680         int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
 681         {                                                               \
 682                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 683                     __RET_ADDR);                                        \
 684                 return (atomic_testandclear_##name(ptr, v));            \
 685         }
 686
 687 #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
 688         int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
 689         {                                                               \
 690                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 691                     __RET_ADDR);                                        \
 692                 return (atomic_testandset_##name(ptr, v));              \
 693         }
 694
 695 #define ASAN_ATOMIC_FUNC_SWAP(name, type)                               \
 696         type kasan_atomic_swap_##name(volatile type *ptr, type val)     \
 697         {                                                               \
 698                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 699                     __RET_ADDR);                                        \
 700                 return (atomic_swap_##name(ptr, val));                  \
 701         }
 702
 703 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 704         int kasan_atomic_cmpset_##name(volatile type *ptr, type oval,   \
 705             type nval)                                                  \
 706         {                                                               \
 707                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 708                     __RET_ADDR);                                        \
 709                 return (atomic_cmpset_##name(ptr, oval, nval));         \
 710         }
 711
 712 #define ASAN_ATOMIC_FUNC_CMPSET(name, type)                             \
 713         _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 714         _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
 715         _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
 716
 717 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 718         int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
 719             type nval)                                                  \
 720         {                                                               \
 721                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 722                     __RET_ADDR);                                        \
 723                 return (atomic_fcmpset_##name(ptr, oval, nval));        \
 724         }
 725
 726 #define ASAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
 727         _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 728         _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
 729         _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
 730
 731 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
 732         void kasan_atomic_thread_fence_##name(void)                     \
 733         {                                                               \
 734                 atomic_thread_fence_##name();                           \
 735         }
 736
 737 #define _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 738         type kasan_atomic_load_##name(volatile type *ptr)               \
 739         {                                                               \
 740                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 741                     __RET_ADDR);                                        \
 742                 return (atomic_load_##name(ptr));                       \
 743         }
 744
 745 #define ASAN_ATOMIC_FUNC_LOAD(name, type)                               \
 746         _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 747         _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
 748
 749 #define _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 750         void kasan_atomic_store_##name(volatile type *ptr, type val)    \
 751         {                                                               \
 752                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 753                     __RET_ADDR);                                        \
 754                 atomic_store_##name(ptr, val);                          \
 755         }
 756
 757 #define ASAN_ATOMIC_FUNC_STORE(name, type)                              \
 758         _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 759         _ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
 760
 761 ASAN_ATOMIC_FUNC_ADD(8, uint8_t);
 762 ASAN_ATOMIC_FUNC_ADD(16, uint16_t);
 763 ASAN_ATOMIC_FUNC_ADD(32, uint32_t);
 764 ASAN_ATOMIC_FUNC_ADD(64, uint64_t);
 765 ASAN_ATOMIC_FUNC_ADD(int, u_int);
 766 ASAN_ATOMIC_FUNC_ADD(long, u_long);
 767 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
 768
 769 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
 770 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
 771 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
 772 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
 773 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
 774 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
 775 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
 776
 777 ASAN_ATOMIC_FUNC_SET(8, uint8_t);
 778 ASAN_ATOMIC_FUNC_SET(16, uint16_t);
 779 ASAN_ATOMIC_FUNC_SET(32, uint32_t);
 780 ASAN_ATOMIC_FUNC_SET(64, uint64_t);
 781 ASAN_ATOMIC_FUNC_SET(int, u_int);
 782 ASAN_ATOMIC_FUNC_SET(long, u_long);
 783 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
 784
 785 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
 786 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
 787 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
 788 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
 789 ASAN_ATOMIC_FUNC_CLEAR(int, u_int);
 790 ASAN_ATOMIC_FUNC_CLEAR(long, u_long);
 791 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
 792
 793 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
 794 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
 795 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int);
 796 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long);
 797
 798 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
 799 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
 800 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
 801 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
 802 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
 803
 804 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
 805 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
 806 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
 807 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
 808
 809 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
 810 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
 811 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
 812 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
 813
 814 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t);
 815 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t);
 816 ASAN_ATOMIC_FUNC_SWAP(int, u_int);
 817 ASAN_ATOMIC_FUNC_SWAP(long, u_long);
 818 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
 819
 820 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
 821 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
 822 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
 823 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
 824 ASAN_ATOMIC_FUNC_CMPSET(int, u_int);
 825 ASAN_ATOMIC_FUNC_CMPSET(long, u_long);
 826 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
 827
 828 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
 829 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
 830 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
 831 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
 832 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int);
 833 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long);
 834 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
 835
 836 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t);
 837 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t);
 838 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t);
 839 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t);
 840 ASAN_ATOMIC_FUNC_LOAD(char, u_char);
 841 ASAN_ATOMIC_FUNC_LOAD(short, u_short);
 842 ASAN_ATOMIC_FUNC_LOAD(int, u_int);
 843 ASAN_ATOMIC_FUNC_LOAD(long, u_long);
 844 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
 845
 846 ASAN_ATOMIC_FUNC_STORE(8, uint8_t);
 847 ASAN_ATOMIC_FUNC_STORE(16, uint16_t);
 848 ASAN_ATOMIC_FUNC_STORE(32, uint32_t);
 849 ASAN_ATOMIC_FUNC_STORE(64, uint64_t);
 850 ASAN_ATOMIC_FUNC_STORE(char, u_char);
 851 ASAN_ATOMIC_FUNC_STORE(short, u_short);
 852 ASAN_ATOMIC_FUNC_STORE(int, u_int);
 853 ASAN_ATOMIC_FUNC_STORE(long, u_long);
 854 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
 855
 856 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq);
 857 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel);
 858 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
 859 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
 860
 861 void
 862 kasan_atomic_interrupt_fence(void)
 863 {
 864 }
 865
 866 /* -------------------------------------------------------------------------- */
 867
 868 #include <sys/bus.h>
 869 #include <machine/bus.h>
 870 #include <sys/bus_san.h>
 871
 872 int
 873 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
 874     int flags, bus_space_handle_t *handlep)
 875 {
 876         return (bus_space_map(tag, hnd, size, flags, handlep));
 877 }
 878
 879 void
 880 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
 881     bus_size_t size)
 882 {
 883         bus_space_unmap(tag, hnd, size);
 884 }
 885
 886 int
 887 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
 888     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
 889 {
 890         return (bus_space_subregion(tag, hnd, offset, size, handlep));
 891 }
 892
 893 void
 894 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
 895     bus_size_t size)
 896 {
 897         bus_space_free(tag, hnd, size);
 898 }
 899
 900 void
 901 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
 902     bus_size_t offset, bus_size_t size, int flags)
 903 {
 904         bus_space_barrier(tag, hnd, offset, size, flags);
 905 }
 906
 907 #define ASAN_BUS_READ_FUNC(func, width, type)                           \
 908         type kasan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
 909             bus_space_handle_t hnd, bus_size_t offset)                  \
 910         {                                                               \
 911                 return (bus_space_read##func##_##width(tag, hnd,        \
 912                     offset));                                           \
 913         }                                                               \
 914
 915 #define ASAN_BUS_READ_PTR_FUNC(func, width, type)                       \
 916         void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
 917             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
 918             bus_size_t count)                                           \
 919         {                                                               \
 920                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 921                     false, __RET_ADDR);                                 \
 922                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
 923                     count);                                             \
 924         }
 925
 926 ASAN_BUS_READ_FUNC(, 1, uint8_t)
 927 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t)
 928 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
 929 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
 930 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
 931 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
 932
 933 ASAN_BUS_READ_FUNC(, 2, uint16_t)
 934 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t)
 935 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
 936 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
 937 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
 938 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
 939
 940 ASAN_BUS_READ_FUNC(, 4, uint32_t)
 941 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t)
 942 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
 943 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
 944 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
 945 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
 946
 947 ASAN_BUS_READ_FUNC(, 8, uint64_t)
 948
 949 #define ASAN_BUS_WRITE_FUNC(func, width, type)                          \
 950         void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \
 951             bus_space_handle_t hnd, bus_size_t offset, type value)      \
 952         {                                                               \
 953                 bus_space_write##func##_##width(tag, hnd, offset, value);\
 954         }                                                               \
 955
 956 #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
 957         void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
 958             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
 959             bus_size_t count)                                           \
 960         {                                                               \
 961                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 962                     true, __RET_ADDR);                                  \
 963                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
 964                     count);                                             \
 965         }
 966
 967 ASAN_BUS_WRITE_FUNC(, 1, uint8_t)
 968 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
 969 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
 970 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
 971 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
 972 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
 973
 974 ASAN_BUS_WRITE_FUNC(, 2, uint16_t)
 975 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
 976 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
 977 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
 978 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
 979 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
 980
 981 ASAN_BUS_WRITE_FUNC(, 4, uint32_t)
 982 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
 983 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
 984 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
 985 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
 986 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
 987
 988 ASAN_BUS_WRITE_FUNC(, 8, uint64_t)
 989
 990 #define ASAN_BUS_SET_FUNC(func, width, type)                            \
 991         void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
 992             bus_space_handle_t hnd, bus_size_t offset, type value,      \
 993             bus_size_t count)                                           \
 994         {                                                               \
 995                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
 996                     count);                                             \
 997         }
 998
 999 ASAN_BUS_SET_FUNC(multi, 1, uint8_t)
1000 ASAN_BUS_SET_FUNC(region, 1, uint8_t)
1001 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1002 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1003
1004 ASAN_BUS_SET_FUNC(multi, 2, uint16_t)
1005 ASAN_BUS_SET_FUNC(region, 2, uint16_t)
1006 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1007 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1008
1009 ASAN_BUS_SET_FUNC(multi, 4, uint32_t)
1010 ASAN_BUS_SET_FUNC(region, 4, uint32_t)
1011 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1012 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1013
1014 /* -------------------------------------------------------------------------- */
1015
1016 void __asan_register_globals(struct __asan_global *, size_t);
1017 void __asan_unregister_globals(struct __asan_global *, size_t);
1018
1019 void
1020 __asan_register_globals(struct __asan_global *globals, size_t n)
1021 {
1022         size_t i;
1023
1024         for (i = 0; i < n; i++) {
1025                 kasan_mark(globals[i].beg, globals[i].size,
1026                     globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
1027         }
1028 }
1029
1030 void
1031 __asan_unregister_globals(struct __asan_global *globals, size_t n)
1032 {
1033         size_t i;
1034
1035         for (i = 0; i < n; i++) {
1036                 kasan_mark(globals[i].beg, globals[i].size_with_redzone,
1037                     globals[i].size_with_redzone, 0);
1038         }
1039 }
1040
1041 #define ASAN_LOAD_STORE(size)                                   \
1042         void __asan_load##size(unsigned long);                  \
1043         void __asan_load##size(unsigned long addr)              \
1044         {                                                       \
1045                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1046         }                                                       \
1047         void __asan_load##size##_noabort(unsigned long);        \
1048         void __asan_load##size##_noabort(unsigned long addr)    \
1049         {                                                       \
1050                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1051         }                                                       \
1052         void __asan_store##size(unsigned long);                 \
1053         void __asan_store##size(unsigned long addr)             \
1054         {                                                       \
1055                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1056         }                                                       \
1057         void __asan_store##size##_noabort(unsigned long);       \
1058         void __asan_store##size##_noabort(unsigned long addr)   \
1059         {                                                       \
1060                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1061         }
1062
1063 ASAN_LOAD_STORE(1);
1064 ASAN_LOAD_STORE(2);
1065 ASAN_LOAD_STORE(4);
1066 ASAN_LOAD_STORE(8);
1067 ASAN_LOAD_STORE(16);
1068
1069 void __asan_loadN(unsigned long, size_t);
1070 void __asan_loadN_noabort(unsigned long, size_t);
1071 void __asan_storeN(unsigned long, size_t);
1072 void __asan_storeN_noabort(unsigned long, size_t);
1073 void __asan_handle_no_return(void);
1074
1075 void
1076 __asan_loadN(unsigned long addr, size_t size)
1077 {
1078         kasan_shadow_check(addr, size, false, __RET_ADDR);
1079 }
1080
1081 void
1082 __asan_loadN_noabort(unsigned long addr, size_t size)
1083 {
1084         kasan_shadow_check(addr, size, false, __RET_ADDR);
1085 }
1086
1087 void
1088 __asan_storeN(unsigned long addr, size_t size)
1089 {
1090         kasan_shadow_check(addr, size, true, __RET_ADDR);
1091 }
1092
1093 void
1094 __asan_storeN_noabort(unsigned long addr, size_t size)
1095 {
1096         kasan_shadow_check(addr, size, true, __RET_ADDR);
1097 }
1098
1099 void
1100 __asan_handle_no_return(void)
1101 {
1102         /* nothing */
1103 }
1104
1105 #define ASAN_SET_SHADOW(byte) \
1106         void __asan_set_shadow_##byte(void *, size_t);                  \
1107         void __asan_set_shadow_##byte(void *addr, size_t size)          \
1108         {                                                               \
1109                 __builtin_memset((void *)addr, 0x##byte, size);         \
1110         }
1111
1112 ASAN_SET_SHADOW(00);
1113 ASAN_SET_SHADOW(f1);
1114 ASAN_SET_SHADOW(f2);
1115 ASAN_SET_SHADOW(f3);
1116 ASAN_SET_SHADOW(f5);
1117 ASAN_SET_SHADOW(f8);
1118
1119 void __asan_poison_stack_memory(const void *, size_t);
1120 void __asan_unpoison_stack_memory(const void *, size_t);
1121
1122 void
1123 __asan_poison_stack_memory(const void *addr, size_t size)
1124 {
1125         size = roundup(size, KASAN_SHADOW_SCALE);
1126         kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1127 }
1128
1129 void
1130 __asan_unpoison_stack_memory(const void *addr, size_t size)
1131 {
1132         kasan_shadow_Nbyte_markvalid(addr, size);
1133 }
1134
1135 void __asan_alloca_poison(const void *, size_t);
1136 void __asan_allocas_unpoison(const void *, const void *);
1137
1138 void
1139 __asan_alloca_poison(const void *addr, size_t size)
1140 {
1141         const void *l, *r;
1142
1143         KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0,
1144             ("%s: invalid address %p", __func__, addr));
1145
1146         l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1147         r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1148
1149         kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1150         kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1151             KASAN_STACK_MID);
1152         kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1153 }
1154
1155 void
1156 __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1157 {
1158         size_t size;
1159
1160         if (__predict_false(!stkbegin))
1161                 return;
1162         if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1163                 return;
1164         size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1165
1166         kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1167 }
1168
1169 void __asan_poison_memory_region(const void *addr, size_t size);
1170 void __asan_unpoison_memory_region(const void *addr, size_t size);
1171
1172 void
1173 __asan_poison_memory_region(const void *addr, size_t size)
1174 {
1175 }
1176
1177 void
1178 __asan_unpoison_memory_region(const void *addr, size_t size)
1179 {
1180 }