sys/kern/subr_asan.c

   1 /*      $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $      */
   2
   3 /*
   4  * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
   5  * All rights reserved.
   6  *
   7  * This code is part of the KASAN subsystem of the NetBSD kernel.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  23  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  25  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  26  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 #define SAN_RUNTIME
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35 #if 0
  36 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $");
  37 #endif
  38
  39 #include <sys/param.h>
  40 #include <sys/systm.h>
  41 #include <sys/asan.h>
  42 #include <sys/kernel.h>
  43 #include <sys/stack.h>
  44 #include <sys/sysctl.h>
  45
  46 #include <machine/asan.h>
  47 #include <machine/bus.h>
  48
  49 /* ASAN constants. Part of the compiler ABI. */
  50 #define KASAN_SHADOW_MASK               (KASAN_SHADOW_SCALE - 1)
  51 #define KASAN_ALLOCA_SCALE_SIZE         32
  52
  53 /* ASAN ABI version. */
  54 #if defined(__clang__) && (__clang_major__ - 0 >= 6)
  55 #define ASAN_ABI_VERSION        8
  56 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
  57 #define ASAN_ABI_VERSION        8
  58 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
  59 #define ASAN_ABI_VERSION        6
  60 #else
  61 #error "Unsupported compiler version"
  62 #endif
  63
  64 #define __RET_ADDR      (unsigned long)__builtin_return_address(0)
  65
  66 /* Global variable descriptor. Part of the compiler ABI.  */
  67 struct __asan_global_source_location {
  68         const char *filename;
  69         int line_no;
  70         int column_no;
  71 };
  72
  73 struct __asan_global {
  74         const void *beg;                /* address of the global variable */
  75         size_t size;                    /* size of the global variable */
  76         size_t size_with_redzone;       /* size with the redzone */
  77         const void *name;               /* name of the variable */
  78         const void *module_name;        /* name of the module where the var is declared */
  79         unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
  80         struct __asan_global_source_location *location;
  81 #if ASAN_ABI_VERSION >= 7
  82         uintptr_t odr_indicator;        /* the address of the ODR indicator symbol */
  83 #endif
  84 };
  85
  86 FEATURE(kasan, "Kernel address sanitizer");
  87
  88 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  89     "KASAN options");
  90
  91 static int panic_on_violation = 1;
  92 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN,
  93     &panic_on_violation, 0,
  94     "Panic if an invalid access is detected");
  95
  96 static bool kasan_enabled __read_mostly = false;
  97
  98 /* -------------------------------------------------------------------------- */
  99
 100 void
 101 kasan_shadow_map(vm_offset_t addr, size_t size)
 102 {
 103         size_t sz, npages, i;
 104         vm_offset_t sva, eva;
 105
 106         KASSERT(addr % KASAN_SHADOW_SCALE == 0,
 107             ("%s: invalid address %#lx", __func__, addr));
 108
 109         sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE;
 110
 111         sva = kasan_md_addr_to_shad(addr);
 112         eva = kasan_md_addr_to_shad(addr) + sz;
 113
 114         sva = rounddown(sva, PAGE_SIZE);
 115         eva = roundup(eva, PAGE_SIZE);
 116
 117         npages = (eva - sva) / PAGE_SIZE;
 118
 119         KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS,
 120             ("%s: invalid address range %#lx-%#lx", __func__, sva, eva));
 121
 122         for (i = 0; i < npages; i++)
 123                 pmap_kasan_enter(sva + ptoa(i));
 124 }
 125
 126 void
 127 kasan_init(void)
 128 {
 129         int disabled;
 130
 131         disabled = 0;
 132         TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled);
 133         if (disabled)
 134                 return;
 135
 136         /* MD initialization. */
 137         kasan_md_init();
 138
 139         /* Now officially enabled. */
 140         kasan_enabled = true;
 141 }
 142
 143 static inline const char *
 144 kasan_code_name(uint8_t code)
 145 {
 146         switch (code) {
 147         case KASAN_GENERIC_REDZONE:
 148                 return "GenericRedZone";
 149         case KASAN_MALLOC_REDZONE:
 150                 return "MallocRedZone";
 151         case KASAN_KMEM_REDZONE:
 152                 return "KmemRedZone";
 153         case KASAN_UMA_FREED:
 154                 return "UMAUseAfterFree";
 155         case KASAN_KSTACK_FREED:
 156                 return "KernelStack";
 157         case KASAN_EXEC_ARGS_FREED:
 158                 return "ExecKVA";
 159         case 1 ... 7:
 160                 return "RedZonePartial";
 161         case KASAN_STACK_LEFT:
 162                 return "StackLeft";
 163         case KASAN_STACK_MID:
 164                 return "StackMiddle";
 165         case KASAN_STACK_RIGHT:
 166                 return "StackRight";
 167         case KASAN_USE_AFTER_RET:
 168                 return "UseAfterRet";
 169         case KASAN_USE_AFTER_SCOPE:
 170                 return "UseAfterScope";
 171         default:
 172                 return "Unknown";
 173         }
 174 }
 175
 176 #define REPORT(f, ...) do {                             \
 177         if (panic_on_violation) {                       \
 178                 kasan_enabled = false;                  \
 179                 panic(f, __VA_ARGS__);                  \
 180         } else {                                        \
 181                 struct stack st;                        \
 182                                                         \
 183                 stack_save(&st);                        \
 184                 printf(f "\n", __VA_ARGS__);            \
 185                 stack_print_ddb(&st);                   \
 186         }                                               \
 187 } while (0)
 188
 189 static void
 190 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
 191     uint8_t code)
 192 {
 193         REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)",
 194             size, (write ? "write" : "read"), addr, kasan_code_name(code),
 195             code);
 196 }
 197
 198 static __always_inline void
 199 kasan_shadow_1byte_markvalid(unsigned long addr)
 200 {
 201         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 202         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 203
 204         *byte = last;
 205 }
 206
 207 static __always_inline void
 208 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
 209 {
 210         size_t i;
 211
 212         for (i = 0; i < size; i++) {
 213                 kasan_shadow_1byte_markvalid((unsigned long)addr + i);
 214         }
 215 }
 216
 217 static __always_inline void
 218 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
 219 {
 220         void *shad;
 221
 222         if (__predict_false(size == 0))
 223                 return;
 224         if (__predict_false(kasan_md_unsupported((vm_offset_t)addr)))
 225                 return;
 226
 227         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 228             ("%s: invalid address %p", __func__, addr));
 229         KASSERT(size % KASAN_SHADOW_SCALE == 0,
 230             ("%s: invalid size %zu", __func__, size));
 231
 232         shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr);
 233         size = size >> KASAN_SHADOW_SCALE_SHIFT;
 234
 235         __builtin_memset(shad, code, size);
 236 }
 237
 238 /*
 239  * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
 240  * and the rest as invalid. There are generally two use cases:
 241  *
 242  *  o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
 243  *    the redzone at the end of the buffer as invalid. If the entire is to be
 244  *    marked invalid, origsize will be 0.
 245  *
 246  *  o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
 247  */
 248 void
 249 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
 250 {
 251         size_t i, n, redz;
 252         int8_t *shad;
 253
 254         if (__predict_false(!kasan_enabled))
 255                 return;
 256
 257         if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS &&
 258             (vm_offset_t)addr < DMAP_MAX_ADDRESS)
 259                 return;
 260
 261         KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS &&
 262             (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS,
 263             ("%s: invalid address %p", __func__, addr));
 264         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 265             ("%s: invalid address %p", __func__, addr));
 266         redz = redzsize - roundup(size, KASAN_SHADOW_SCALE);
 267         KASSERT(redz % KASAN_SHADOW_SCALE == 0,
 268             ("%s: invalid size %zu", __func__, redz));
 269         shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr);
 270
 271         /* Chunks of 8 bytes, valid. */
 272         n = size / KASAN_SHADOW_SCALE;
 273         for (i = 0; i < n; i++) {
 274                 *shad++ = 0;
 275         }
 276
 277         /* Possibly one chunk, mid. */
 278         if ((size & KASAN_SHADOW_MASK) != 0) {
 279                 *shad++ = (size & KASAN_SHADOW_MASK);
 280         }
 281
 282         /* Chunks of 8 bytes, invalid. */
 283         n = redz / KASAN_SHADOW_SCALE;
 284         for (i = 0; i < n; i++) {
 285                 *shad++ = code;
 286         }
 287 }
 288
 289 /* -------------------------------------------------------------------------- */
 290
 291 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size)                 \
 292         (addr >> KASAN_SHADOW_SCALE_SHIFT) !=                   \
 293             ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
 294
 295 static __always_inline bool
 296 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
 297 {
 298         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 299         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 300
 301         if (__predict_true(*byte == 0 || last <= *byte)) {
 302                 return (true);
 303         }
 304         *code = *byte;
 305         return (false);
 306 }
 307
 308 static __always_inline bool
 309 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
 310 {
 311         int8_t *byte, last;
 312
 313         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
 314                 return (kasan_shadow_1byte_isvalid(addr, code) &&
 315                     kasan_shadow_1byte_isvalid(addr+1, code));
 316         }
 317
 318         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 319         last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
 320
 321         if (__predict_true(*byte == 0 || last <= *byte)) {
 322                 return (true);
 323         }
 324         *code = *byte;
 325         return (false);
 326 }
 327
 328 static __always_inline bool
 329 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
 330 {
 331         int8_t *byte, last;
 332
 333         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
 334                 return (kasan_shadow_2byte_isvalid(addr, code) &&
 335                     kasan_shadow_2byte_isvalid(addr+2, code));
 336         }
 337
 338         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 339         last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
 340
 341         if (__predict_true(*byte == 0 || last <= *byte)) {
 342                 return (true);
 343         }
 344         *code = *byte;
 345         return (false);
 346 }
 347
 348 static __always_inline bool
 349 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
 350 {
 351         int8_t *byte, last;
 352
 353         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
 354                 return (kasan_shadow_4byte_isvalid(addr, code) &&
 355                     kasan_shadow_4byte_isvalid(addr+4, code));
 356         }
 357
 358         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 359         last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
 360
 361         if (__predict_true(*byte == 0 || last <= *byte)) {
 362                 return (true);
 363         }
 364         *code = *byte;
 365         return (false);
 366 }
 367
 368 static __always_inline bool
 369 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
 370 {
 371         size_t i;
 372
 373         for (i = 0; i < size; i++) {
 374                 if (!kasan_shadow_1byte_isvalid(addr+i, code))
 375                         return (false);
 376         }
 377
 378         return (true);
 379 }
 380
 381 static __always_inline void
 382 kasan_shadow_check(unsigned long addr, size_t size, bool write,
 383     unsigned long retaddr)
 384 {
 385         uint8_t code;
 386         bool valid;
 387
 388         if (__predict_false(!kasan_enabled))
 389                 return;
 390         if (__predict_false(size == 0))
 391                 return;
 392         if (__predict_false(kasan_md_unsupported(addr)))
 393                 return;
 394         if (KERNEL_PANICKED())
 395                 return;
 396
 397         if (__builtin_constant_p(size)) {
 398                 switch (size) {
 399                 case 1:
 400                         valid = kasan_shadow_1byte_isvalid(addr, &code);
 401                         break;
 402                 case 2:
 403                         valid = kasan_shadow_2byte_isvalid(addr, &code);
 404                         break;
 405                 case 4:
 406                         valid = kasan_shadow_4byte_isvalid(addr, &code);
 407                         break;
 408                 case 8:
 409                         valid = kasan_shadow_8byte_isvalid(addr, &code);
 410                         break;
 411                 default:
 412                         valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 413                         break;
 414                 }
 415         } else {
 416                 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 417         }
 418
 419         if (__predict_false(!valid)) {
 420                 kasan_report(addr, size, write, retaddr, code);
 421         }
 422 }
 423
 424 /* -------------------------------------------------------------------------- */
 425
 426 void *
 427 kasan_memcpy(void *dst, const void *src, size_t len)
 428 {
 429         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 430         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 431         return (__builtin_memcpy(dst, src, len));
 432 }
 433
 434 int
 435 kasan_memcmp(const void *b1, const void *b2, size_t len)
 436 {
 437         kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
 438         kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
 439         return (__builtin_memcmp(b1, b2, len));
 440 }
 441
 442 void *
 443 kasan_memset(void *b, int c, size_t len)
 444 {
 445         kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
 446         return (__builtin_memset(b, c, len));
 447 }
 448
 449 void *
 450 kasan_memmove(void *dst, const void *src, size_t len)
 451 {
 452         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 453         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 454         return (__builtin_memmove(dst, src, len));
 455 }
 456
 457 size_t
 458 kasan_strlen(const char *str)
 459 {
 460         const char *s;
 461
 462         s = str;
 463         while (1) {
 464                 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
 465                 if (*s == '\0')
 466                         break;
 467                 s++;
 468         }
 469
 470         return (s - str);
 471 }
 472
 473 char *
 474 kasan_strcpy(char *dst, const char *src)
 475 {
 476         char *save = dst;
 477
 478         while (1) {
 479                 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
 480                 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
 481                 *dst = *src;
 482                 if (*src == '\0')
 483                         break;
 484                 src++, dst++;
 485         }
 486
 487         return save;
 488 }
 489
 490 int
 491 kasan_strcmp(const char *s1, const char *s2)
 492 {
 493         while (1) {
 494                 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
 495                 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
 496                 if (*s1 != *s2)
 497                         break;
 498                 if (*s1 == '\0')
 499                         return 0;
 500                 s1++, s2++;
 501         }
 502
 503         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 504 }
 505
 506 int
 507 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
 508 {
 509         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 510         return (copyin(uaddr, kaddr, len));
 511 }
 512
 513 int
 514 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 515 {
 516         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 517         return (copyinstr(uaddr, kaddr, len, done));
 518 }
 519
 520 int
 521 kasan_copyout(const void *kaddr, void *uaddr, size_t len)
 522 {
 523         kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
 524         return (copyout(kaddr, uaddr, len));
 525 }
 526
 527 /* -------------------------------------------------------------------------- */
 528
 529 #include <machine/atomic.h>
 530 #include <sys/atomic_san.h>
 531
 532 #define _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 533         void kasan_atomic_add_##name(volatile type *ptr, type val)      \
 534         {                                                               \
 535                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 536                     __RET_ADDR);                                        \
 537                 atomic_add_##name(ptr, val);                            \
 538         }
 539
 540 #define ASAN_ATOMIC_FUNC_ADD(name, type)                                \
 541         _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 542         _ASAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
 543         _ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
 544
 545 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 546         void kasan_atomic_subtract_##name(volatile type *ptr, type val) \
 547         {                                                               \
 548                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 549                     __RET_ADDR);                                        \
 550                 atomic_subtract_##name(ptr, val);                       \
 551         }
 552
 553 #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
 554         _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 555         _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
 556         _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
 557
 558 #define _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 559         void kasan_atomic_set_##name(volatile type *ptr, type val)      \
 560         {                                                               \
 561                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 562                     __RET_ADDR);                                        \
 563                 atomic_set_##name(ptr, val);                            \
 564         }
 565
 566 #define ASAN_ATOMIC_FUNC_SET(name, type)                                \
 567         _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 568         _ASAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
 569         _ASAN_ATOMIC_FUNC_SET(rel_##name, type)
 570
 571 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 572         void kasan_atomic_clear_##name(volatile type *ptr, type val)    \
 573         {                                                               \
 574                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 575                     __RET_ADDR);                                        \
 576                 atomic_clear_##name(ptr, val);                          \
 577         }
 578
 579 #define ASAN_ATOMIC_FUNC_CLEAR(name, type)                              \
 580         _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 581         _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
 582         _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
 583
 584 #define ASAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
 585         type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \
 586         {                                                               \
 587                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 588                     __RET_ADDR);                                        \
 589                 return (atomic_fetchadd_##name(ptr, val));              \
 590         }
 591
 592 #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
 593         type kasan_atomic_readandclear_##name(volatile type *ptr)       \
 594         {                                                               \
 595                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 596                     __RET_ADDR);                                        \
 597                 return (atomic_readandclear_##name(ptr));               \
 598         }
 599
 600 #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
 601         int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
 602         {                                                               \
 603                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 604                     __RET_ADDR);                                        \
 605                 return (atomic_testandclear_##name(ptr, v));            \
 606         }
 607
 608 #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
 609         int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
 610         {                                                               \
 611                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 612                     __RET_ADDR);                                        \
 613                 return (atomic_testandset_##name(ptr, v));              \
 614         }
 615
 616 #define ASAN_ATOMIC_FUNC_SWAP(name, type)                               \
 617         type kasan_atomic_swap_##name(volatile type *ptr, type val)     \
 618         {                                                               \
 619                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 620                     __RET_ADDR);                                        \
 621                 return (atomic_swap_##name(ptr, val));                  \
 622         }
 623
 624 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 625         int kasan_atomic_cmpset_##name(volatile type *ptr, type oval,   \
 626             type nval)                                                  \
 627         {                                                               \
 628                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 629                     __RET_ADDR);                                        \
 630                 return (atomic_cmpset_##name(ptr, oval, nval));         \
 631         }
 632
 633 #define ASAN_ATOMIC_FUNC_CMPSET(name, type)                             \
 634         _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 635         _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
 636         _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
 637
 638 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 639         int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
 640             type nval)                                                  \
 641         {                                                               \
 642                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 643                     __RET_ADDR);                                        \
 644                 return (atomic_fcmpset_##name(ptr, oval, nval));        \
 645         }
 646
 647 #define ASAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
 648         _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 649         _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
 650         _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
 651
 652 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
 653         void kasan_atomic_thread_fence_##name(void)                     \
 654         {                                                               \
 655                 atomic_thread_fence_##name();                           \
 656         }
 657
 658 #define _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 659         type kasan_atomic_load_##name(volatile type *ptr)               \
 660         {                                                               \
 661                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 662                     __RET_ADDR);                                        \
 663                 return (atomic_load_##name(ptr));                       \
 664         }
 665
 666 #define ASAN_ATOMIC_FUNC_LOAD(name, type)                               \
 667         _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 668         _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
 669
 670 #define _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 671         void kasan_atomic_store_##name(volatile type *ptr, type val)    \
 672         {                                                               \
 673                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 674                     __RET_ADDR);                                        \
 675                 atomic_store_##name(ptr, val);                          \
 676         }
 677
 678 #define ASAN_ATOMIC_FUNC_STORE(name, type)                              \
 679         _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 680         _ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
 681
 682 ASAN_ATOMIC_FUNC_ADD(8, uint8_t);
 683 ASAN_ATOMIC_FUNC_ADD(16, uint16_t);
 684 ASAN_ATOMIC_FUNC_ADD(32, uint32_t);
 685 ASAN_ATOMIC_FUNC_ADD(64, uint64_t);
 686 ASAN_ATOMIC_FUNC_ADD(int, u_int);
 687 ASAN_ATOMIC_FUNC_ADD(long, u_long);
 688 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
 689
 690 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
 691 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
 692 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
 693 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
 694 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
 695 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
 696 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
 697
 698 ASAN_ATOMIC_FUNC_SET(8, uint8_t);
 699 ASAN_ATOMIC_FUNC_SET(16, uint16_t);
 700 ASAN_ATOMIC_FUNC_SET(32, uint32_t);
 701 ASAN_ATOMIC_FUNC_SET(64, uint64_t);
 702 ASAN_ATOMIC_FUNC_SET(int, u_int);
 703 ASAN_ATOMIC_FUNC_SET(long, u_long);
 704 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
 705
 706 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
 707 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
 708 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
 709 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
 710 ASAN_ATOMIC_FUNC_CLEAR(int, u_int);
 711 ASAN_ATOMIC_FUNC_CLEAR(long, u_long);
 712 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
 713
 714 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
 715 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
 716 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int);
 717 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long);
 718
 719 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
 720 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
 721 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
 722 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
 723 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
 724
 725 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
 726 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
 727 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
 728 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
 729
 730 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
 731 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
 732 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
 733 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
 734
 735 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t);
 736 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t);
 737 ASAN_ATOMIC_FUNC_SWAP(int, u_int);
 738 ASAN_ATOMIC_FUNC_SWAP(long, u_long);
 739 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
 740
 741 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
 742 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
 743 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
 744 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
 745 ASAN_ATOMIC_FUNC_CMPSET(int, u_int);
 746 ASAN_ATOMIC_FUNC_CMPSET(long, u_long);
 747 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
 748
 749 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
 750 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
 751 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
 752 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
 753 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int);
 754 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long);
 755 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
 756
 757 _ASAN_ATOMIC_FUNC_LOAD(bool, bool);
 758 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t);
 759 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t);
 760 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t);
 761 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t);
 762 ASAN_ATOMIC_FUNC_LOAD(char, u_char);
 763 ASAN_ATOMIC_FUNC_LOAD(short, u_short);
 764 ASAN_ATOMIC_FUNC_LOAD(int, u_int);
 765 ASAN_ATOMIC_FUNC_LOAD(long, u_long);
 766 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
 767
 768 _ASAN_ATOMIC_FUNC_STORE(bool, bool);
 769 ASAN_ATOMIC_FUNC_STORE(8, uint8_t);
 770 ASAN_ATOMIC_FUNC_STORE(16, uint16_t);
 771 ASAN_ATOMIC_FUNC_STORE(32, uint32_t);
 772 ASAN_ATOMIC_FUNC_STORE(64, uint64_t);
 773 ASAN_ATOMIC_FUNC_STORE(char, u_char);
 774 ASAN_ATOMIC_FUNC_STORE(short, u_short);
 775 ASAN_ATOMIC_FUNC_STORE(int, u_int);
 776 ASAN_ATOMIC_FUNC_STORE(long, u_long);
 777 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
 778
 779 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq);
 780 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel);
 781 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
 782 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
 783
 784 void
 785 kasan_atomic_interrupt_fence(void)
 786 {
 787 }
 788
 789 /* -------------------------------------------------------------------------- */
 790
 791 #include <sys/bus.h>
 792 #include <machine/bus.h>
 793 #include <sys/bus_san.h>
 794
 795 int
 796 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
 797     int flags, bus_space_handle_t *handlep)
 798 {
 799         return (bus_space_map(tag, hnd, size, flags, handlep));
 800 }
 801
 802 void
 803 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
 804     bus_size_t size)
 805 {
 806         bus_space_unmap(tag, hnd, size);
 807 }
 808
 809 int
 810 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
 811     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
 812 {
 813         return (bus_space_subregion(tag, hnd, offset, size, handlep));
 814 }
 815
 816 void
 817 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
 818     bus_size_t size)
 819 {
 820         bus_space_free(tag, hnd, size);
 821 }
 822
 823 void
 824 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
 825     bus_size_t offset, bus_size_t size, int flags)
 826 {
 827         bus_space_barrier(tag, hnd, offset, size, flags);
 828 }
 829
 830 #define ASAN_BUS_READ_FUNC(func, width, type)                           \
 831         type kasan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
 832             bus_space_handle_t hnd, bus_size_t offset)                  \
 833         {                                                               \
 834                 return (bus_space_read##func##_##width(tag, hnd,        \
 835                     offset));                                           \
 836         }                                                               \
 837
 838 #define ASAN_BUS_READ_PTR_FUNC(func, width, type)                       \
 839         void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
 840             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
 841             bus_size_t count)                                           \
 842         {                                                               \
 843                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 844                     false, __RET_ADDR);                                 \
 845                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
 846                     count);                                             \
 847         }
 848
 849 ASAN_BUS_READ_FUNC(, 1, uint8_t)
 850 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t)
 851 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
 852 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
 853 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
 854 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
 855
 856 ASAN_BUS_READ_FUNC(, 2, uint16_t)
 857 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t)
 858 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
 859 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
 860 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
 861 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
 862
 863 ASAN_BUS_READ_FUNC(, 4, uint32_t)
 864 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t)
 865 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
 866 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
 867 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
 868 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
 869
 870 ASAN_BUS_READ_FUNC(, 8, uint64_t)
 871
 872 #define ASAN_BUS_WRITE_FUNC(func, width, type)                          \
 873         void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \
 874             bus_space_handle_t hnd, bus_size_t offset, type value)      \
 875         {                                                               \
 876                 bus_space_write##func##_##width(tag, hnd, offset, value);\
 877         }                                                               \
 878
 879 #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
 880         void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
 881             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
 882             bus_size_t count)                                           \
 883         {                                                               \
 884                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 885                     true, __RET_ADDR);                                  \
 886                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
 887                     count);                                             \
 888         }
 889
 890 ASAN_BUS_WRITE_FUNC(, 1, uint8_t)
 891 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
 892 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
 893 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
 894 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
 895 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
 896
 897 ASAN_BUS_WRITE_FUNC(, 2, uint16_t)
 898 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
 899 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
 900 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
 901 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
 902 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
 903
 904 ASAN_BUS_WRITE_FUNC(, 4, uint32_t)
 905 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
 906 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
 907 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
 908 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
 909 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
 910
 911 ASAN_BUS_WRITE_FUNC(, 8, uint64_t)
 912
 913 #define ASAN_BUS_SET_FUNC(func, width, type)                            \
 914         void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
 915             bus_space_handle_t hnd, bus_size_t offset, type value,      \
 916             bus_size_t count)                                           \
 917         {                                                               \
 918                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
 919                     count);                                             \
 920         }
 921
 922 ASAN_BUS_SET_FUNC(multi, 1, uint8_t)
 923 ASAN_BUS_SET_FUNC(region, 1, uint8_t)
 924 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
 925 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
 926
 927 ASAN_BUS_SET_FUNC(multi, 2, uint16_t)
 928 ASAN_BUS_SET_FUNC(region, 2, uint16_t)
 929 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
 930 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
 931
 932 ASAN_BUS_SET_FUNC(multi, 4, uint32_t)
 933 ASAN_BUS_SET_FUNC(region, 4, uint32_t)
 934 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
 935 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
 936
 937 #define ASAN_BUS_PEEK_FUNC(width, type)                                 \
 938         int kasan_bus_space_peek_##width(bus_space_tag_t tag,           \
 939             bus_space_handle_t hnd, bus_size_t offset, type *valuep)    \
 940         {                                                               \
 941                 return (bus_space_peek_##width(tag, hnd, offset,        \
 942                     valuep));                                           \
 943         }
 944
 945 ASAN_BUS_PEEK_FUNC(1, uint8_t)
 946 ASAN_BUS_PEEK_FUNC(2, uint16_t)
 947 ASAN_BUS_PEEK_FUNC(4, uint32_t)
 948 ASAN_BUS_PEEK_FUNC(8, uint64_t)
 949
 950 #define ASAN_BUS_POKE_FUNC(width, type)                                 \
 951         int kasan_bus_space_poke_##width(bus_space_tag_t tag,           \
 952             bus_space_handle_t hnd, bus_size_t offset, type value)      \
 953         {                                                               \
 954                 return (bus_space_poke_##width(tag, hnd, offset,        \
 955                     value));                                            \
 956         }
 957
 958 ASAN_BUS_POKE_FUNC(1, uint8_t)
 959 ASAN_BUS_POKE_FUNC(2, uint16_t)
 960 ASAN_BUS_POKE_FUNC(4, uint32_t)
 961 ASAN_BUS_POKE_FUNC(8, uint64_t)
 962
 963 /* -------------------------------------------------------------------------- */
 964
 965 void __asan_register_globals(struct __asan_global *, size_t);
 966 void __asan_unregister_globals(struct __asan_global *, size_t);
 967
 968 void
 969 __asan_register_globals(struct __asan_global *globals, size_t n)
 970 {
 971         size_t i;
 972
 973         for (i = 0; i < n; i++) {
 974                 kasan_mark(globals[i].beg, globals[i].size,
 975                     globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
 976         }
 977 }
 978
 979 void
 980 __asan_unregister_globals(struct __asan_global *globals, size_t n)
 981 {
 982         size_t i;
 983
 984         for (i = 0; i < n; i++) {
 985                 kasan_mark(globals[i].beg, globals[i].size_with_redzone,
 986                     globals[i].size_with_redzone, 0);
 987         }
 988 }
 989
 990 #define ASAN_LOAD_STORE(size)                                   \
 991         void __asan_load##size(unsigned long);                  \
 992         void __asan_load##size(unsigned long addr)              \
 993         {                                                       \
 994                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
 995         }                                                       \
 996         void __asan_load##size##_noabort(unsigned long);        \
 997         void __asan_load##size##_noabort(unsigned long addr)    \
 998         {                                                       \
 999                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1000         }                                                       \
1001         void __asan_store##size(unsigned long);                 \
1002         void __asan_store##size(unsigned long addr)             \
1003         {                                                       \
1004                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1005         }                                                       \
1006         void __asan_store##size##_noabort(unsigned long);       \
1007         void __asan_store##size##_noabort(unsigned long addr)   \
1008         {                                                       \
1009                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1010         }
1011
1012 ASAN_LOAD_STORE(1);
1013 ASAN_LOAD_STORE(2);
1014 ASAN_LOAD_STORE(4);
1015 ASAN_LOAD_STORE(8);
1016 ASAN_LOAD_STORE(16);
1017
1018 void __asan_loadN(unsigned long, size_t);
1019 void __asan_loadN_noabort(unsigned long, size_t);
1020 void __asan_storeN(unsigned long, size_t);
1021 void __asan_storeN_noabort(unsigned long, size_t);
1022 void __asan_handle_no_return(void);
1023
1024 void
1025 __asan_loadN(unsigned long addr, size_t size)
1026 {
1027         kasan_shadow_check(addr, size, false, __RET_ADDR);
1028 }
1029
1030 void
1031 __asan_loadN_noabort(unsigned long addr, size_t size)
1032 {
1033         kasan_shadow_check(addr, size, false, __RET_ADDR);
1034 }
1035
1036 void
1037 __asan_storeN(unsigned long addr, size_t size)
1038 {
1039         kasan_shadow_check(addr, size, true, __RET_ADDR);
1040 }
1041
1042 void
1043 __asan_storeN_noabort(unsigned long addr, size_t size)
1044 {
1045         kasan_shadow_check(addr, size, true, __RET_ADDR);
1046 }
1047
1048 void
1049 __asan_handle_no_return(void)
1050 {
1051         /* nothing */
1052 }
1053
1054 #define ASAN_SET_SHADOW(byte) \
1055         void __asan_set_shadow_##byte(void *, size_t);                  \
1056         void __asan_set_shadow_##byte(void *addr, size_t size)          \
1057         {                                                               \
1058                 __builtin_memset((void *)addr, 0x##byte, size);         \
1059         }
1060
1061 ASAN_SET_SHADOW(00);
1062 ASAN_SET_SHADOW(f1);
1063 ASAN_SET_SHADOW(f2);
1064 ASAN_SET_SHADOW(f3);
1065 ASAN_SET_SHADOW(f5);
1066 ASAN_SET_SHADOW(f8);
1067
1068 void __asan_poison_stack_memory(const void *, size_t);
1069 void __asan_unpoison_stack_memory(const void *, size_t);
1070
1071 void
1072 __asan_poison_stack_memory(const void *addr, size_t size)
1073 {
1074         size = roundup(size, KASAN_SHADOW_SCALE);
1075         kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1076 }
1077
1078 void
1079 __asan_unpoison_stack_memory(const void *addr, size_t size)
1080 {
1081         kasan_shadow_Nbyte_markvalid(addr, size);
1082 }
1083
1084 void __asan_alloca_poison(const void *, size_t);
1085 void __asan_allocas_unpoison(const void *, const void *);
1086
1087 void
1088 __asan_alloca_poison(const void *addr, size_t size)
1089 {
1090         const void *l, *r;
1091
1092         KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0,
1093             ("%s: invalid address %p", __func__, addr));
1094
1095         l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1096         r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1097
1098         kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1099         kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1100             KASAN_STACK_MID);
1101         kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1102 }
1103
1104 void
1105 __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1106 {
1107         size_t size;
1108
1109         if (__predict_false(!stkbegin))
1110                 return;
1111         if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1112                 return;
1113         size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1114
1115         kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1116 }
1117
1118 void __asan_poison_memory_region(const void *addr, size_t size);
1119 void __asan_unpoison_memory_region(const void *addr, size_t size);
1120
1121 void
1122 __asan_poison_memory_region(const void *addr, size_t size)
1123 {
1124 }
1125
1126 void
1127 __asan_unpoison_memory_region(const void *addr, size_t size)
1128 {
1129 }