sys/kern/subr_asan.c

   1 /*      $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $      */
   2
   3 /*
   4  * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
   5  * All rights reserved.
   6  *
   7  * This code is part of the KASAN subsystem of the NetBSD kernel.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  23  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  25  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  26  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 #define SAN_RUNTIME
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35 #if 0
  36 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $");
  37 #endif
  38
  39 #include <sys/param.h>
  40 #include <sys/systm.h>
  41 #include <sys/asan.h>
  42 #include <sys/kernel.h>
  43 #include <sys/stack.h>
  44 #include <sys/sysctl.h>
  45
  46 #include <machine/asan.h>
  47
  48 /* ASAN constants. Part of the compiler ABI. */
  49 #define KASAN_SHADOW_MASK               (KASAN_SHADOW_SCALE - 1)
  50 #define KASAN_ALLOCA_SCALE_SIZE         32
  51
  52 /* ASAN ABI version. */
  53 #if defined(__clang__) && (__clang_major__ - 0 >= 6)
  54 #define ASAN_ABI_VERSION        8
  55 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
  56 #define ASAN_ABI_VERSION        8
  57 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
  58 #define ASAN_ABI_VERSION        6
  59 #else
  60 #error "Unsupported compiler version"
  61 #endif
  62
  63 #define __RET_ADDR      (unsigned long)__builtin_return_address(0)
  64
  65 /* Global variable descriptor. Part of the compiler ABI.  */
  66 struct __asan_global_source_location {
  67         const char *filename;
  68         int line_no;
  69         int column_no;
  70 };
  71
  72 struct __asan_global {
  73         const void *beg;                /* address of the global variable */
  74         size_t size;                    /* size of the global variable */
  75         size_t size_with_redzone;       /* size with the redzone */
  76         const void *name;               /* name of the variable */
  77         const void *module_name;        /* name of the module where the var is declared */
  78         unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
  79         struct __asan_global_source_location *location;
  80 #if ASAN_ABI_VERSION >= 7
  81         uintptr_t odr_indicator;        /* the address of the ODR indicator symbol */
  82 #endif
  83 };
  84
  85 FEATURE(kasan, "Kernel address sanitizer");
  86
  87 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  88     "KASAN options");
  89
  90 static int panic_on_violation = 1;
  91 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN,
  92     &panic_on_violation, 0,
  93     "Panic if an invalid access is detected");
  94
  95 static bool kasan_enabled __read_mostly = false;
  96
  97 /* -------------------------------------------------------------------------- */
  98
  99 void
 100 kasan_shadow_map(vm_offset_t addr, size_t size)
 101 {
 102         size_t sz, npages, i;
 103         vm_offset_t sva, eva;
 104
 105         KASSERT(addr % KASAN_SHADOW_SCALE == 0,
 106             ("%s: invalid address %#lx", __func__, addr));
 107
 108         sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE;
 109
 110         sva = kasan_md_addr_to_shad(addr);
 111         eva = kasan_md_addr_to_shad(addr) + sz;
 112
 113         sva = rounddown(sva, PAGE_SIZE);
 114         eva = roundup(eva, PAGE_SIZE);
 115
 116         npages = (eva - sva) / PAGE_SIZE;
 117
 118         KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS,
 119             ("%s: invalid address range %#lx-%#lx", __func__, sva, eva));
 120
 121         for (i = 0; i < npages; i++)
 122                 pmap_kasan_enter(sva + ptoa(i));
 123 }
 124
 125 void
 126 kasan_init(void)
 127 {
 128         int disabled;
 129
 130         disabled = 0;
 131         TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled);
 132         if (disabled)
 133                 return;
 134
 135         /* MD initialization. */
 136         kasan_md_init();
 137
 138         /* Now officially enabled. */
 139         kasan_enabled = true;
 140 }
 141
 142 static inline const char *
 143 kasan_code_name(uint8_t code)
 144 {
 145         switch (code) {
 146         case KASAN_GENERIC_REDZONE:
 147                 return "GenericRedZone";
 148         case KASAN_MALLOC_REDZONE:
 149                 return "MallocRedZone";
 150         case KASAN_KMEM_REDZONE:
 151                 return "KmemRedZone";
 152         case KASAN_UMA_FREED:
 153                 return "UMAUseAfterFree";
 154         case KASAN_KSTACK_FREED:
 155                 return "KernelStack";
 156         case KASAN_EXEC_ARGS_FREED:
 157                 return "ExecKVA";
 158         case 1 ... 7:
 159                 return "RedZonePartial";
 160         case KASAN_STACK_LEFT:
 161                 return "StackLeft";
 162         case KASAN_STACK_MID:
 163                 return "StackMiddle";
 164         case KASAN_STACK_RIGHT:
 165                 return "StackRight";
 166         case KASAN_USE_AFTER_RET:
 167                 return "UseAfterRet";
 168         case KASAN_USE_AFTER_SCOPE:
 169                 return "UseAfterScope";
 170         default:
 171                 return "Unknown";
 172         }
 173 }
 174
 175 #define REPORT(f, ...) do {                             \
 176         if (panic_on_violation) {                       \
 177                 panic(f, __VA_ARGS__);                  \
 178         } else {                                        \
 179                 struct stack st;                        \
 180                                                         \
 181                 stack_save(&st);                        \
 182                 printf(f "\n", __VA_ARGS__);            \
 183                 stack_print_ddb(&st);                   \
 184         }                                               \
 185 } while (0)
 186
 187 static void
 188 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
 189     uint8_t code)
 190 {
 191         REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)",
 192             size, (write ? "write" : "read"), addr, kasan_code_name(code),
 193             code);
 194 }
 195
 196 static __always_inline void
 197 kasan_shadow_1byte_markvalid(unsigned long addr)
 198 {
 199         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 200         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 201
 202         *byte = last;
 203 }
 204
 205 static __always_inline void
 206 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
 207 {
 208         size_t i;
 209
 210         for (i = 0; i < size; i++) {
 211                 kasan_shadow_1byte_markvalid((unsigned long)addr + i);
 212         }
 213 }
 214
 215 static __always_inline void
 216 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
 217 {
 218         void *shad;
 219
 220         if (__predict_false(size == 0))
 221                 return;
 222         if (__predict_false(kasan_md_unsupported((vm_offset_t)addr)))
 223                 return;
 224
 225         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 226             ("%s: invalid address %p", __func__, addr));
 227         KASSERT(size % KASAN_SHADOW_SCALE == 0,
 228             ("%s: invalid size %zu", __func__, size));
 229
 230         shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr);
 231         size = size >> KASAN_SHADOW_SCALE_SHIFT;
 232
 233         __builtin_memset(shad, code, size);
 234 }
 235
 236 /*
 237  * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
 238  * and the rest as invalid. There are generally two use cases:
 239  *
 240  *  o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
 241  *    the redzone at the end of the buffer as invalid. If the entire is to be
 242  *    marked invalid, origsize will be 0.
 243  *
 244  *  o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
 245  */
 246 void
 247 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
 248 {
 249         size_t i, n, redz;
 250         int8_t *shad;
 251
 252         if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS &&
 253             (vm_offset_t)addr < DMAP_MAX_ADDRESS)
 254                 return;
 255
 256         KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS &&
 257             (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS,
 258             ("%s: invalid address %p", __func__, addr));
 259         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 260             ("%s: invalid address %p", __func__, addr));
 261         redz = redzsize - roundup(size, KASAN_SHADOW_SCALE);
 262         KASSERT(redz % KASAN_SHADOW_SCALE == 0,
 263             ("%s: invalid size %zu", __func__, redz));
 264         shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr);
 265
 266         /* Chunks of 8 bytes, valid. */
 267         n = size / KASAN_SHADOW_SCALE;
 268         for (i = 0; i < n; i++) {
 269                 *shad++ = 0;
 270         }
 271
 272         /* Possibly one chunk, mid. */
 273         if ((size & KASAN_SHADOW_MASK) != 0) {
 274                 *shad++ = (size & KASAN_SHADOW_MASK);
 275         }
 276
 277         /* Chunks of 8 bytes, invalid. */
 278         n = redz / KASAN_SHADOW_SCALE;
 279         for (i = 0; i < n; i++) {
 280                 *shad++ = code;
 281         }
 282 }
 283
 284 /* -------------------------------------------------------------------------- */
 285
 286 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size)                 \
 287         (addr >> KASAN_SHADOW_SCALE_SHIFT) !=                   \
 288             ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
 289
 290 static __always_inline bool
 291 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
 292 {
 293         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 294         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 295
 296         if (__predict_true(*byte == 0 || last <= *byte)) {
 297                 return (true);
 298         }
 299         *code = *byte;
 300         return (false);
 301 }
 302
 303 static __always_inline bool
 304 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
 305 {
 306         int8_t *byte, last;
 307
 308         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
 309                 return (kasan_shadow_1byte_isvalid(addr, code) &&
 310                     kasan_shadow_1byte_isvalid(addr+1, code));
 311         }
 312
 313         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 314         last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
 315
 316         if (__predict_true(*byte == 0 || last <= *byte)) {
 317                 return (true);
 318         }
 319         *code = *byte;
 320         return (false);
 321 }
 322
 323 static __always_inline bool
 324 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
 325 {
 326         int8_t *byte, last;
 327
 328         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
 329                 return (kasan_shadow_2byte_isvalid(addr, code) &&
 330                     kasan_shadow_2byte_isvalid(addr+2, code));
 331         }
 332
 333         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 334         last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
 335
 336         if (__predict_true(*byte == 0 || last <= *byte)) {
 337                 return (true);
 338         }
 339         *code = *byte;
 340         return (false);
 341 }
 342
 343 static __always_inline bool
 344 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
 345 {
 346         int8_t *byte, last;
 347
 348         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
 349                 return (kasan_shadow_4byte_isvalid(addr, code) &&
 350                     kasan_shadow_4byte_isvalid(addr+4, code));
 351         }
 352
 353         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 354         last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
 355
 356         if (__predict_true(*byte == 0 || last <= *byte)) {
 357                 return (true);
 358         }
 359         *code = *byte;
 360         return (false);
 361 }
 362
 363 static __always_inline bool
 364 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
 365 {
 366         size_t i;
 367
 368         for (i = 0; i < size; i++) {
 369                 if (!kasan_shadow_1byte_isvalid(addr+i, code))
 370                         return (false);
 371         }
 372
 373         return (true);
 374 }
 375
 376 static __always_inline void
 377 kasan_shadow_check(unsigned long addr, size_t size, bool write,
 378     unsigned long retaddr)
 379 {
 380         uint8_t code;
 381         bool valid;
 382
 383         if (__predict_false(!kasan_enabled))
 384                 return;
 385         if (__predict_false(size == 0))
 386                 return;
 387         if (__predict_false(kasan_md_unsupported(addr)))
 388                 return;
 389         if (__predict_false(panicstr != NULL))
 390                 return;
 391
 392         if (__builtin_constant_p(size)) {
 393                 switch (size) {
 394                 case 1:
 395                         valid = kasan_shadow_1byte_isvalid(addr, &code);
 396                         break;
 397                 case 2:
 398                         valid = kasan_shadow_2byte_isvalid(addr, &code);
 399                         break;
 400                 case 4:
 401                         valid = kasan_shadow_4byte_isvalid(addr, &code);
 402                         break;
 403                 case 8:
 404                         valid = kasan_shadow_8byte_isvalid(addr, &code);
 405                         break;
 406                 default:
 407                         valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 408                         break;
 409                 }
 410         } else {
 411                 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 412         }
 413
 414         if (__predict_false(!valid)) {
 415                 kasan_report(addr, size, write, retaddr, code);
 416         }
 417 }
 418
 419 /* -------------------------------------------------------------------------- */
 420
 421 void *
 422 kasan_memcpy(void *dst, const void *src, size_t len)
 423 {
 424         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 425         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 426         return (__builtin_memcpy(dst, src, len));
 427 }
 428
 429 int
 430 kasan_memcmp(const void *b1, const void *b2, size_t len)
 431 {
 432         kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
 433         kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
 434         return (__builtin_memcmp(b1, b2, len));
 435 }
 436
 437 void *
 438 kasan_memset(void *b, int c, size_t len)
 439 {
 440         kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
 441         return (__builtin_memset(b, c, len));
 442 }
 443
 444 void *
 445 kasan_memmove(void *dst, const void *src, size_t len)
 446 {
 447         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 448         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 449         return (__builtin_memmove(dst, src, len));
 450 }
 451
 452 size_t
 453 kasan_strlen(const char *str)
 454 {
 455         const char *s;
 456
 457         s = str;
 458         while (1) {
 459                 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
 460                 if (*s == '\0')
 461                         break;
 462                 s++;
 463         }
 464
 465         return (s - str);
 466 }
 467
 468 char *
 469 kasan_strcpy(char *dst, const char *src)
 470 {
 471         char *save = dst;
 472
 473         while (1) {
 474                 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
 475                 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
 476                 *dst = *src;
 477                 if (*src == '\0')
 478                         break;
 479                 src++, dst++;
 480         }
 481
 482         return save;
 483 }
 484
 485 int
 486 kasan_strcmp(const char *s1, const char *s2)
 487 {
 488         while (1) {
 489                 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
 490                 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
 491                 if (*s1 != *s2)
 492                         break;
 493                 if (*s1 == '\0')
 494                         return 0;
 495                 s1++, s2++;
 496         }
 497
 498         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 499 }
 500
 501 int
 502 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
 503 {
 504         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 505         return (copyin(uaddr, kaddr, len));
 506 }
 507
 508 int
 509 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 510 {
 511         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 512         return (copyinstr(uaddr, kaddr, len, done));
 513 }
 514
 515 int
 516 kasan_copyout(const void *kaddr, void *uaddr, size_t len)
 517 {
 518         kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
 519         return (copyout(kaddr, uaddr, len));
 520 }
 521
 522 /* -------------------------------------------------------------------------- */
 523
 524 #include <machine/atomic.h>
 525 #define ATOMIC_SAN_PREFIX       kasan
 526 #include <sys/atomic_san.h>
 527
 528 #define _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 529         void kasan_atomic_add_##name(volatile type *ptr, type val)      \
 530         {                                                               \
 531                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 532                     __RET_ADDR);                                        \
 533                 atomic_add_##name(ptr, val);                            \
 534         }
 535
 536 #define ASAN_ATOMIC_FUNC_ADD(name, type)                                \
 537         _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 538         _ASAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
 539         _ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
 540
 541 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 542         void kasan_atomic_subtract_##name(volatile type *ptr, type val) \
 543         {                                                               \
 544                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 545                     __RET_ADDR);                                        \
 546                 atomic_subtract_##name(ptr, val);                       \
 547         }
 548
 549 #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
 550         _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 551         _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
 552         _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
 553
 554 #define _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 555         void kasan_atomic_set_##name(volatile type *ptr, type val)      \
 556         {                                                               \
 557                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 558                     __RET_ADDR);                                        \
 559                 atomic_set_##name(ptr, val);                            \
 560         }
 561
 562 #define ASAN_ATOMIC_FUNC_SET(name, type)                                \
 563         _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 564         _ASAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
 565         _ASAN_ATOMIC_FUNC_SET(rel_##name, type)
 566
 567 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 568         void kasan_atomic_clear_##name(volatile type *ptr, type val)    \
 569         {                                                               \
 570                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 571                     __RET_ADDR);                                        \
 572                 atomic_clear_##name(ptr, val);                          \
 573         }
 574
 575 #define ASAN_ATOMIC_FUNC_CLEAR(name, type)                              \
 576         _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 577         _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
 578         _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
 579
 580 #define ASAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
 581         type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \
 582         {                                                               \
 583                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 584                     __RET_ADDR);                                        \
 585                 return (atomic_fetchadd_##name(ptr, val));              \
 586         }
 587
 588 #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
 589         type kasan_atomic_readandclear_##name(volatile type *ptr)       \
 590         {                                                               \
 591                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 592                     __RET_ADDR);                                        \
 593                 return (atomic_readandclear_##name(ptr));               \
 594         }
 595
 596 #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
 597         int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
 598         {                                                               \
 599                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 600                     __RET_ADDR);                                        \
 601                 return (atomic_testandclear_##name(ptr, v));            \
 602         }
 603
 604 #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
 605         int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
 606         {                                                               \
 607                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 608                     __RET_ADDR);                                        \
 609                 return (atomic_testandset_##name(ptr, v));              \
 610         }
 611
 612 #define ASAN_ATOMIC_FUNC_SWAP(name, type)                               \
 613         type kasan_atomic_swap_##name(volatile type *ptr, type val)     \
 614         {                                                               \
 615                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 616                     __RET_ADDR);                                        \
 617                 return (atomic_swap_##name(ptr, val));                  \
 618         }
 619
 620 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 621         int kasan_atomic_cmpset_##name(volatile type *ptr, type oval,   \
 622             type nval)                                                  \
 623         {                                                               \
 624                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 625                     __RET_ADDR);                                        \
 626                 return (atomic_cmpset_##name(ptr, oval, nval));         \
 627         }
 628
 629 #define ASAN_ATOMIC_FUNC_CMPSET(name, type)                             \
 630         _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 631         _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
 632         _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
 633
 634 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 635         int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
 636             type nval)                                                  \
 637         {                                                               \
 638                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 639                     __RET_ADDR);                                        \
 640                 return (atomic_fcmpset_##name(ptr, oval, nval));        \
 641         }
 642
 643 #define ASAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
 644         _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 645         _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
 646         _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
 647
 648 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
 649         void kasan_atomic_thread_fence_##name(void)                     \
 650         {                                                               \
 651                 atomic_thread_fence_##name();                           \
 652         }
 653
 654 #define _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 655         type kasan_atomic_load_##name(volatile type *ptr)               \
 656         {                                                               \
 657                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 658                     __RET_ADDR);                                        \
 659                 return (atomic_load_##name(ptr));                       \
 660         }
 661
 662 #define ASAN_ATOMIC_FUNC_LOAD(name, type)                               \
 663         _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 664         _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
 665
 666 #define _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 667         void kasan_atomic_store_##name(volatile type *ptr, type val)    \
 668         {                                                               \
 669                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 670                     __RET_ADDR);                                        \
 671                 atomic_store_##name(ptr, val);                          \
 672         }
 673
 674 #define ASAN_ATOMIC_FUNC_STORE(name, type)                              \
 675         _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 676         _ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
 677
 678 ASAN_ATOMIC_FUNC_ADD(8, uint8_t);
 679 ASAN_ATOMIC_FUNC_ADD(16, uint16_t);
 680 ASAN_ATOMIC_FUNC_ADD(32, uint32_t);
 681 ASAN_ATOMIC_FUNC_ADD(64, uint64_t);
 682 ASAN_ATOMIC_FUNC_ADD(int, u_int);
 683 ASAN_ATOMIC_FUNC_ADD(long, u_long);
 684 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
 685
 686 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
 687 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
 688 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
 689 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
 690 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
 691 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
 692 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
 693
 694 ASAN_ATOMIC_FUNC_SET(8, uint8_t);
 695 ASAN_ATOMIC_FUNC_SET(16, uint16_t);
 696 ASAN_ATOMIC_FUNC_SET(32, uint32_t);
 697 ASAN_ATOMIC_FUNC_SET(64, uint64_t);
 698 ASAN_ATOMIC_FUNC_SET(int, u_int);
 699 ASAN_ATOMIC_FUNC_SET(long, u_long);
 700 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
 701
 702 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
 703 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
 704 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
 705 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
 706 ASAN_ATOMIC_FUNC_CLEAR(int, u_int);
 707 ASAN_ATOMIC_FUNC_CLEAR(long, u_long);
 708 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
 709
 710 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
 711 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
 712 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int);
 713 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long);
 714
 715 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
 716 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
 717 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
 718 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
 719 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
 720
 721 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
 722 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
 723 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
 724 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
 725
 726 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
 727 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
 728 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
 729 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
 730
 731 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t);
 732 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t);
 733 ASAN_ATOMIC_FUNC_SWAP(int, u_int);
 734 ASAN_ATOMIC_FUNC_SWAP(long, u_long);
 735 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
 736
 737 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
 738 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
 739 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
 740 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
 741 ASAN_ATOMIC_FUNC_CMPSET(int, u_int);
 742 ASAN_ATOMIC_FUNC_CMPSET(long, u_long);
 743 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
 744
 745 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
 746 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
 747 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
 748 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
 749 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int);
 750 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long);
 751 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
 752
 753 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t);
 754 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t);
 755 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t);
 756 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t);
 757 ASAN_ATOMIC_FUNC_LOAD(char, u_char);
 758 ASAN_ATOMIC_FUNC_LOAD(short, u_short);
 759 ASAN_ATOMIC_FUNC_LOAD(int, u_int);
 760 ASAN_ATOMIC_FUNC_LOAD(long, u_long);
 761 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
 762
 763 ASAN_ATOMIC_FUNC_STORE(8, uint8_t);
 764 ASAN_ATOMIC_FUNC_STORE(16, uint16_t);
 765 ASAN_ATOMIC_FUNC_STORE(32, uint32_t);
 766 ASAN_ATOMIC_FUNC_STORE(64, uint64_t);
 767 ASAN_ATOMIC_FUNC_STORE(char, u_char);
 768 ASAN_ATOMIC_FUNC_STORE(short, u_short);
 769 ASAN_ATOMIC_FUNC_STORE(int, u_int);
 770 ASAN_ATOMIC_FUNC_STORE(long, u_long);
 771 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
 772
 773 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq);
 774 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel);
 775 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
 776 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
 777
 778 void
 779 kasan_atomic_interrupt_fence(void)
 780 {
 781 }
 782
 783 /* -------------------------------------------------------------------------- */
 784
 785 #include <sys/bus.h>
 786 #include <machine/bus.h>
 787 #define BUS_SAN_PREFIX  kasan
 788 #include <sys/bus_san.h>
 789
 790 int
 791 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
 792     int flags, bus_space_handle_t *handlep)
 793 {
 794         return (bus_space_map(tag, hnd, size, flags, handlep));
 795 }
 796
 797 void
 798 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
 799     bus_size_t size)
 800 {
 801         bus_space_unmap(tag, hnd, size);
 802 }
 803
 804 int
 805 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
 806     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
 807 {
 808         return (bus_space_subregion(tag, hnd, offset, size, handlep));
 809 }
 810
 811 void
 812 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
 813     bus_size_t size)
 814 {
 815         bus_space_free(tag, hnd, size);
 816 }
 817
 818 void
 819 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
 820     bus_size_t offset, bus_size_t size, int flags)
 821 {
 822         bus_space_barrier(tag, hnd, offset, size, flags);
 823 }
 824
 825 #define ASAN_BUS_READ_FUNC(func, width, type)                           \
 826         type kasan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
 827             bus_space_handle_t hnd, bus_size_t offset)                  \
 828         {                                                               \
 829                 return (bus_space_read##func##_##width(tag, hnd,        \
 830                     offset));                                           \
 831         }                                                               \
 832
 833 #define ASAN_BUS_READ_PTR_FUNC(func, width, type)                       \
 834         void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
 835             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
 836             bus_size_t count)                                           \
 837         {                                                               \
 838                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 839                     false, __RET_ADDR);                                 \
 840                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
 841                     count);                                             \
 842         }
 843
 844 ASAN_BUS_READ_FUNC(, 1, uint8_t)
 845 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t)
 846 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
 847 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
 848 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
 849 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
 850
 851 ASAN_BUS_READ_FUNC(, 2, uint16_t)
 852 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t)
 853 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
 854 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
 855 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
 856 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
 857
 858 ASAN_BUS_READ_FUNC(, 4, uint32_t)
 859 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t)
 860 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
 861 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
 862 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
 863 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
 864
 865 ASAN_BUS_READ_FUNC(, 8, uint64_t)
 866
 867 #define ASAN_BUS_WRITE_FUNC(func, width, type)                          \
 868         void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \
 869             bus_space_handle_t hnd, bus_size_t offset, type value)      \
 870         {                                                               \
 871                 bus_space_write##func##_##width(tag, hnd, offset, value);\
 872         }                                                               \
 873
 874 #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
 875         void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
 876             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
 877             bus_size_t count)                                           \
 878         {                                                               \
 879                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 880                     true, __RET_ADDR);                                  \
 881                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
 882                     count);                                             \
 883         }
 884
 885 ASAN_BUS_WRITE_FUNC(, 1, uint8_t)
 886 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
 887 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
 888 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
 889 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
 890 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
 891
 892 ASAN_BUS_WRITE_FUNC(, 2, uint16_t)
 893 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
 894 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
 895 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
 896 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
 897 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
 898
 899 ASAN_BUS_WRITE_FUNC(, 4, uint32_t)
 900 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
 901 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
 902 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
 903 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
 904 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
 905
 906 ASAN_BUS_WRITE_FUNC(, 8, uint64_t)
 907
 908 #define ASAN_BUS_SET_FUNC(func, width, type)                            \
 909         void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
 910             bus_space_handle_t hnd, bus_size_t offset, type value,      \
 911             bus_size_t count)                                           \
 912         {                                                               \
 913                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
 914                     count);                                             \
 915         }
 916
 917 ASAN_BUS_SET_FUNC(multi, 1, uint8_t)
 918 ASAN_BUS_SET_FUNC(region, 1, uint8_t)
 919 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
 920 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
 921
 922 ASAN_BUS_SET_FUNC(multi, 2, uint16_t)
 923 ASAN_BUS_SET_FUNC(region, 2, uint16_t)
 924 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
 925 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
 926
 927 ASAN_BUS_SET_FUNC(multi, 4, uint32_t)
 928 ASAN_BUS_SET_FUNC(region, 4, uint32_t)
 929 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
 930 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
 931
 932 /* -------------------------------------------------------------------------- */
 933
 934 void __asan_register_globals(struct __asan_global *, size_t);
 935 void __asan_unregister_globals(struct __asan_global *, size_t);
 936
 937 void
 938 __asan_register_globals(struct __asan_global *globals, size_t n)
 939 {
 940         size_t i;
 941
 942         for (i = 0; i < n; i++) {
 943                 kasan_mark(globals[i].beg, globals[i].size,
 944                     globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
 945         }
 946 }
 947
 948 void
 949 __asan_unregister_globals(struct __asan_global *globals, size_t n)
 950 {
 951         /* never called */
 952 }
 953
 954 #define ASAN_LOAD_STORE(size)                                   \
 955         void __asan_load##size(unsigned long);                  \
 956         void __asan_load##size(unsigned long addr)              \
 957         {                                                       \
 958                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
 959         }                                                       \
 960         void __asan_load##size##_noabort(unsigned long);        \
 961         void __asan_load##size##_noabort(unsigned long addr)    \
 962         {                                                       \
 963                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
 964         }                                                       \
 965         void __asan_store##size(unsigned long);                 \
 966         void __asan_store##size(unsigned long addr)             \
 967         {                                                       \
 968                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
 969         }                                                       \
 970         void __asan_store##size##_noabort(unsigned long);       \
 971         void __asan_store##size##_noabort(unsigned long addr)   \
 972         {                                                       \
 973                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
 974         }
 975
 976 ASAN_LOAD_STORE(1);
 977 ASAN_LOAD_STORE(2);
 978 ASAN_LOAD_STORE(4);
 979 ASAN_LOAD_STORE(8);
 980 ASAN_LOAD_STORE(16);
 981
 982 void __asan_loadN(unsigned long, size_t);
 983 void __asan_loadN_noabort(unsigned long, size_t);
 984 void __asan_storeN(unsigned long, size_t);
 985 void __asan_storeN_noabort(unsigned long, size_t);
 986 void __asan_handle_no_return(void);
 987
 988 void
 989 __asan_loadN(unsigned long addr, size_t size)
 990 {
 991         kasan_shadow_check(addr, size, false, __RET_ADDR);
 992 }
 993
 994 void
 995 __asan_loadN_noabort(unsigned long addr, size_t size)
 996 {
 997         kasan_shadow_check(addr, size, false, __RET_ADDR);
 998 }
 999
1000 void
1001 __asan_storeN(unsigned long addr, size_t size)
1002 {
1003         kasan_shadow_check(addr, size, true, __RET_ADDR);
1004 }
1005
1006 void
1007 __asan_storeN_noabort(unsigned long addr, size_t size)
1008 {
1009         kasan_shadow_check(addr, size, true, __RET_ADDR);
1010 }
1011
1012 void
1013 __asan_handle_no_return(void)
1014 {
1015         /* nothing */
1016 }
1017
1018 #define ASAN_SET_SHADOW(byte) \
1019         void __asan_set_shadow_##byte(void *, size_t);                  \
1020         void __asan_set_shadow_##byte(void *addr, size_t size)          \
1021         {                                                               \
1022                 __builtin_memset((void *)addr, 0x##byte, size);         \
1023         }
1024
1025 ASAN_SET_SHADOW(00);
1026 ASAN_SET_SHADOW(f1);
1027 ASAN_SET_SHADOW(f2);
1028 ASAN_SET_SHADOW(f3);
1029 ASAN_SET_SHADOW(f5);
1030 ASAN_SET_SHADOW(f8);
1031
1032 void __asan_poison_stack_memory(const void *, size_t);
1033 void __asan_unpoison_stack_memory(const void *, size_t);
1034
1035 void
1036 __asan_poison_stack_memory(const void *addr, size_t size)
1037 {
1038         size = roundup(size, KASAN_SHADOW_SCALE);
1039         kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1040 }
1041
1042 void
1043 __asan_unpoison_stack_memory(const void *addr, size_t size)
1044 {
1045         kasan_shadow_Nbyte_markvalid(addr, size);
1046 }
1047
1048 void __asan_alloca_poison(const void *, size_t);
1049 void __asan_allocas_unpoison(const void *, const void *);
1050
1051 void
1052 __asan_alloca_poison(const void *addr, size_t size)
1053 {
1054         const void *l, *r;
1055
1056         KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0,
1057             ("%s: invalid address %p", __func__, addr));
1058
1059         l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1060         r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1061
1062         kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1063         kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1064             KASAN_STACK_MID);
1065         kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1066 }
1067
1068 void
1069 __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1070 {
1071         size_t size;
1072
1073         if (__predict_false(!stkbegin))
1074                 return;
1075         if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1076                 return;
1077         size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1078
1079         kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1080 }
1081
1082 void __asan_poison_memory_region(const void *addr, size_t size);
1083 void __asan_unpoison_memory_region(const void *addr, size_t size);
1084
1085 void
1086 __asan_poison_memory_region(const void *addr, size_t size)
1087 {
1088 }
1089
1090 void
1091 __asan_unpoison_memory_region(const void *addr, size_t size)
1092 {
1093 }