sys/kern/subr_asan.c

   1 /*      $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $      */
   2
   3 /*
   4  * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
   5  * All rights reserved.
   6  *
   7  * This code is part of the KASAN subsystem of the NetBSD kernel.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  23  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  25  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  26  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 #define SAN_RUNTIME
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35 #if 0
  36 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $");
  37 #endif
  38
  39 #include <sys/param.h>
  40 #include <sys/systm.h>
  41 #include <sys/asan.h>
  42 #include <sys/kernel.h>
  43 #include <sys/stack.h>
  44 #include <sys/sysctl.h>
  45
  46 #include <machine/asan.h>
  47 #include <machine/bus.h>
  48
  49 /* ASAN constants. Part of the compiler ABI. */
  50 #define KASAN_SHADOW_MASK               (KASAN_SHADOW_SCALE - 1)
  51 #define KASAN_ALLOCA_SCALE_SIZE         32
  52
  53 /* ASAN ABI version. */
  54 #if defined(__clang__) && (__clang_major__ - 0 >= 6)
  55 #define ASAN_ABI_VERSION        8
  56 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
  57 #define ASAN_ABI_VERSION        8
  58 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
  59 #define ASAN_ABI_VERSION        6
  60 #else
  61 #error "Unsupported compiler version"
  62 #endif
  63
  64 #define __RET_ADDR      (unsigned long)__builtin_return_address(0)
  65
  66 /* Global variable descriptor. Part of the compiler ABI.  */
  67 struct __asan_global_source_location {
  68         const char *filename;
  69         int line_no;
  70         int column_no;
  71 };
  72
  73 struct __asan_global {
  74         const void *beg;                /* address of the global variable */
  75         size_t size;                    /* size of the global variable */
  76         size_t size_with_redzone;       /* size with the redzone */
  77         const void *name;               /* name of the variable */
  78         const void *module_name;        /* name of the module where the var is declared */
  79         unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
  80         struct __asan_global_source_location *location;
  81 #if ASAN_ABI_VERSION >= 7
  82         uintptr_t odr_indicator;        /* the address of the ODR indicator symbol */
  83 #endif
  84 };
  85
  86 FEATURE(kasan, "Kernel address sanitizer");
  87
  88 static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  89     "KASAN options");
  90
  91 static int panic_on_violation = 1;
  92 SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN,
  93     &panic_on_violation, 0,
  94     "Panic if an invalid access is detected");
  95
  96 static bool kasan_enabled __read_mostly = false;
  97
  98 /* -------------------------------------------------------------------------- */
  99
 100 void
 101 kasan_shadow_map(vm_offset_t addr, size_t size)
 102 {
 103         size_t sz, npages, i;
 104         vm_offset_t sva, eva;
 105
 106         KASSERT(addr % KASAN_SHADOW_SCALE == 0,
 107             ("%s: invalid address %#lx", __func__, addr));
 108
 109         sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE;
 110
 111         sva = kasan_md_addr_to_shad(addr);
 112         eva = kasan_md_addr_to_shad(addr) + sz;
 113
 114         sva = rounddown(sva, PAGE_SIZE);
 115         eva = roundup(eva, PAGE_SIZE);
 116
 117         npages = (eva - sva) / PAGE_SIZE;
 118
 119         KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS,
 120             ("%s: invalid address range %#lx-%#lx", __func__, sva, eva));
 121
 122         for (i = 0; i < npages; i++)
 123                 pmap_san_enter(sva + ptoa(i));
 124 }
 125
 126 void
 127 kasan_init(void)
 128 {
 129         int disabled;
 130
 131         disabled = 0;
 132         TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled);
 133         if (disabled)
 134                 return;
 135
 136         /* MD initialization. */
 137         kasan_md_init();
 138
 139         /* Now officially enabled. */
 140         kasan_enabled = true;
 141 }
 142
 143 void
 144 kasan_init_early(vm_offset_t stack, size_t size)
 145 {
 146         kasan_md_init_early(stack, size);
 147 }
 148
 149 static inline const char *
 150 kasan_code_name(uint8_t code)
 151 {
 152         switch (code) {
 153         case KASAN_GENERIC_REDZONE:
 154                 return "GenericRedZone";
 155         case KASAN_MALLOC_REDZONE:
 156                 return "MallocRedZone";
 157         case KASAN_KMEM_REDZONE:
 158                 return "KmemRedZone";
 159         case KASAN_UMA_FREED:
 160                 return "UMAUseAfterFree";
 161         case KASAN_KSTACK_FREED:
 162                 return "KernelStack";
 163         case KASAN_EXEC_ARGS_FREED:
 164                 return "ExecKVA";
 165         case 1 ... 7:
 166                 return "RedZonePartial";
 167         case KASAN_STACK_LEFT:
 168                 return "StackLeft";
 169         case KASAN_STACK_MID:
 170                 return "StackMiddle";
 171         case KASAN_STACK_RIGHT:
 172                 return "StackRight";
 173         case KASAN_USE_AFTER_RET:
 174                 return "UseAfterRet";
 175         case KASAN_USE_AFTER_SCOPE:
 176                 return "UseAfterScope";
 177         default:
 178                 return "Unknown";
 179         }
 180 }
 181
 182 #define REPORT(f, ...) do {                             \
 183         if (panic_on_violation) {                       \
 184                 kasan_enabled = false;                  \
 185                 panic(f, __VA_ARGS__);                  \
 186         } else {                                        \
 187                 struct stack st;                        \
 188                                                         \
 189                 stack_save(&st);                        \
 190                 printf(f "\n", __VA_ARGS__);            \
 191                 stack_print_ddb(&st);                   \
 192         }                                               \
 193 } while (0)
 194
 195 static void
 196 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
 197     uint8_t code)
 198 {
 199         REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)",
 200             size, (write ? "write" : "read"), addr, kasan_code_name(code),
 201             code);
 202 }
 203
 204 static __always_inline void
 205 kasan_shadow_1byte_markvalid(unsigned long addr)
 206 {
 207         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 208         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 209
 210         *byte = last;
 211 }
 212
 213 static __always_inline void
 214 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
 215 {
 216         size_t i;
 217
 218         for (i = 0; i < size; i++) {
 219                 kasan_shadow_1byte_markvalid((unsigned long)addr + i);
 220         }
 221 }
 222
 223 static __always_inline void
 224 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
 225 {
 226         void *shad;
 227
 228         if (__predict_false(size == 0))
 229                 return;
 230         if (__predict_false(kasan_md_unsupported((vm_offset_t)addr)))
 231                 return;
 232
 233         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 234             ("%s: invalid address %p", __func__, addr));
 235         KASSERT(size % KASAN_SHADOW_SCALE == 0,
 236             ("%s: invalid size %zu", __func__, size));
 237
 238         shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr);
 239         size = size >> KASAN_SHADOW_SCALE_SHIFT;
 240
 241         __builtin_memset(shad, code, size);
 242 }
 243
 244 /*
 245  * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
 246  * and the rest as invalid. There are generally two use cases:
 247  *
 248  *  o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
 249  *    the redzone at the end of the buffer as invalid. If the entire is to be
 250  *    marked invalid, origsize will be 0.
 251  *
 252  *  o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
 253  */
 254 void
 255 kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
 256 {
 257         size_t i, n, redz;
 258         int8_t *shad;
 259
 260         if (__predict_false(!kasan_enabled))
 261                 return;
 262
 263         if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS &&
 264             (vm_offset_t)addr < DMAP_MAX_ADDRESS)
 265                 return;
 266
 267         KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS &&
 268             (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS,
 269             ("%s: invalid address %p", __func__, addr));
 270         KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0,
 271             ("%s: invalid address %p", __func__, addr));
 272         redz = redzsize - roundup(size, KASAN_SHADOW_SCALE);
 273         KASSERT(redz % KASAN_SHADOW_SCALE == 0,
 274             ("%s: invalid size %zu", __func__, redz));
 275         shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr);
 276
 277         /* Chunks of 8 bytes, valid. */
 278         n = size / KASAN_SHADOW_SCALE;
 279         for (i = 0; i < n; i++) {
 280                 *shad++ = 0;
 281         }
 282
 283         /* Possibly one chunk, mid. */
 284         if ((size & KASAN_SHADOW_MASK) != 0) {
 285                 *shad++ = (size & KASAN_SHADOW_MASK);
 286         }
 287
 288         /* Chunks of 8 bytes, invalid. */
 289         n = redz / KASAN_SHADOW_SCALE;
 290         for (i = 0; i < n; i++) {
 291                 *shad++ = code;
 292         }
 293 }
 294
 295 /* -------------------------------------------------------------------------- */
 296
 297 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size)                 \
 298         (addr >> KASAN_SHADOW_SCALE_SHIFT) !=                   \
 299             ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
 300
 301 static __always_inline bool
 302 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
 303 {
 304         int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr);
 305         int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
 306
 307         if (__predict_true(*byte == 0 || last <= *byte)) {
 308                 return (true);
 309         }
 310         *code = *byte;
 311         return (false);
 312 }
 313
 314 static __always_inline bool
 315 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
 316 {
 317         int8_t *byte, last;
 318
 319         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
 320                 return (kasan_shadow_1byte_isvalid(addr, code) &&
 321                     kasan_shadow_1byte_isvalid(addr+1, code));
 322         }
 323
 324         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 325         last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
 326
 327         if (__predict_true(*byte == 0 || last <= *byte)) {
 328                 return (true);
 329         }
 330         *code = *byte;
 331         return (false);
 332 }
 333
 334 static __always_inline bool
 335 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
 336 {
 337         int8_t *byte, last;
 338
 339         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
 340                 return (kasan_shadow_2byte_isvalid(addr, code) &&
 341                     kasan_shadow_2byte_isvalid(addr+2, code));
 342         }
 343
 344         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 345         last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
 346
 347         if (__predict_true(*byte == 0 || last <= *byte)) {
 348                 return (true);
 349         }
 350         *code = *byte;
 351         return (false);
 352 }
 353
 354 static __always_inline bool
 355 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
 356 {
 357         int8_t *byte, last;
 358
 359         if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
 360                 return (kasan_shadow_4byte_isvalid(addr, code) &&
 361                     kasan_shadow_4byte_isvalid(addr+4, code));
 362         }
 363
 364         byte = (int8_t *)kasan_md_addr_to_shad(addr);
 365         last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
 366
 367         if (__predict_true(*byte == 0 || last <= *byte)) {
 368                 return (true);
 369         }
 370         *code = *byte;
 371         return (false);
 372 }
 373
 374 static __always_inline bool
 375 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
 376 {
 377         size_t i;
 378
 379         for (i = 0; i < size; i++) {
 380                 if (!kasan_shadow_1byte_isvalid(addr+i, code))
 381                         return (false);
 382         }
 383
 384         return (true);
 385 }
 386
 387 static __always_inline void
 388 kasan_shadow_check(unsigned long addr, size_t size, bool write,
 389     unsigned long retaddr)
 390 {
 391         uint8_t code;
 392         bool valid;
 393
 394         if (__predict_false(!kasan_enabled))
 395                 return;
 396         if (__predict_false(size == 0))
 397                 return;
 398         if (__predict_false(kasan_md_unsupported(addr)))
 399                 return;
 400         if (KERNEL_PANICKED())
 401                 return;
 402
 403         if (__builtin_constant_p(size)) {
 404                 switch (size) {
 405                 case 1:
 406                         valid = kasan_shadow_1byte_isvalid(addr, &code);
 407                         break;
 408                 case 2:
 409                         valid = kasan_shadow_2byte_isvalid(addr, &code);
 410                         break;
 411                 case 4:
 412                         valid = kasan_shadow_4byte_isvalid(addr, &code);
 413                         break;
 414                 case 8:
 415                         valid = kasan_shadow_8byte_isvalid(addr, &code);
 416                         break;
 417                 default:
 418                         valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 419                         break;
 420                 }
 421         } else {
 422                 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
 423         }
 424
 425         if (__predict_false(!valid)) {
 426                 kasan_report(addr, size, write, retaddr, code);
 427         }
 428 }
 429
 430 /* -------------------------------------------------------------------------- */
 431
 432 void *
 433 kasan_memcpy(void *dst, const void *src, size_t len)
 434 {
 435         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 436         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 437         return (__builtin_memcpy(dst, src, len));
 438 }
 439
 440 int
 441 kasan_memcmp(const void *b1, const void *b2, size_t len)
 442 {
 443         kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
 444         kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
 445         return (__builtin_memcmp(b1, b2, len));
 446 }
 447
 448 void *
 449 kasan_memset(void *b, int c, size_t len)
 450 {
 451         kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
 452         return (__builtin_memset(b, c, len));
 453 }
 454
 455 void *
 456 kasan_memmove(void *dst, const void *src, size_t len)
 457 {
 458         kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
 459         kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
 460         return (__builtin_memmove(dst, src, len));
 461 }
 462
 463 size_t
 464 kasan_strlen(const char *str)
 465 {
 466         const char *s;
 467
 468         s = str;
 469         while (1) {
 470                 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
 471                 if (*s == '\0')
 472                         break;
 473                 s++;
 474         }
 475
 476         return (s - str);
 477 }
 478
 479 char *
 480 kasan_strcpy(char *dst, const char *src)
 481 {
 482         char *save = dst;
 483
 484         while (1) {
 485                 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
 486                 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
 487                 *dst = *src;
 488                 if (*src == '\0')
 489                         break;
 490                 src++, dst++;
 491         }
 492
 493         return save;
 494 }
 495
 496 int
 497 kasan_strcmp(const char *s1, const char *s2)
 498 {
 499         while (1) {
 500                 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
 501                 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
 502                 if (*s1 != *s2)
 503                         break;
 504                 if (*s1 == '\0')
 505                         return 0;
 506                 s1++, s2++;
 507         }
 508
 509         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 510 }
 511
 512 int
 513 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
 514 {
 515         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 516         return (copyin(uaddr, kaddr, len));
 517 }
 518
 519 int
 520 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 521 {
 522         kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
 523         return (copyinstr(uaddr, kaddr, len, done));
 524 }
 525
 526 int
 527 kasan_copyout(const void *kaddr, void *uaddr, size_t len)
 528 {
 529         kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
 530         return (copyout(kaddr, uaddr, len));
 531 }
 532
 533 /* -------------------------------------------------------------------------- */
 534
 535 int
 536 kasan_fubyte(volatile const void *base)
 537 {
 538         return (fubyte(base));
 539 }
 540
 541 int
 542 kasan_fuword16(volatile const void *base)
 543 {
 544         return (fuword16(base));
 545 }
 546
 547 int
 548 kasan_fueword(volatile const void *base, long *val)
 549 {
 550         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 551         return (fueword(base, val));
 552 }
 553
 554 int
 555 kasan_fueword32(volatile const void *base, int32_t *val)
 556 {
 557         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 558         return (fueword32(base, val));
 559 }
 560
 561 int
 562 kasan_fueword64(volatile const void *base, int64_t *val)
 563 {
 564         kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR);
 565         return (fueword64(base, val));
 566 }
 567
 568 int
 569 kasan_subyte(volatile void *base, int byte)
 570 {
 571         return (subyte(base, byte));
 572 }
 573
 574 int
 575 kasan_suword(volatile void *base, long word)
 576 {
 577         return (suword(base, word));
 578 }
 579
 580 int
 581 kasan_suword16(volatile void *base, int word)
 582 {
 583         return (suword16(base, word));
 584 }
 585
 586 int
 587 kasan_suword32(volatile void *base, int32_t word)
 588 {
 589         return (suword32(base, word));
 590 }
 591
 592 int
 593 kasan_suword64(volatile void *base, int64_t word)
 594 {
 595         return (suword64(base, word));
 596 }
 597
 598 int
 599 kasan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
 600     uint32_t newval)
 601 {
 602         kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
 603             __RET_ADDR);
 604         return (casueword32(base, oldval, oldvalp, newval));
 605 }
 606
 607 int
 608 kasan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
 609     u_long newval)
 610 {
 611         kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true,
 612             __RET_ADDR);
 613         return (casueword(base, oldval, oldvalp, newval));
 614 }
 615
 616 /* -------------------------------------------------------------------------- */
 617
 618 #include <machine/atomic.h>
 619 #include <sys/atomic_san.h>
 620
 621 #define _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 622         void kasan_atomic_add_##name(volatile type *ptr, type val)      \
 623         {                                                               \
 624                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 625                     __RET_ADDR);                                        \
 626                 atomic_add_##name(ptr, val);                            \
 627         }
 628
 629 #define ASAN_ATOMIC_FUNC_ADD(name, type)                                \
 630         _ASAN_ATOMIC_FUNC_ADD(name, type)                               \
 631         _ASAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
 632         _ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
 633
 634 #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 635         void kasan_atomic_subtract_##name(volatile type *ptr, type val) \
 636         {                                                               \
 637                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 638                     __RET_ADDR);                                        \
 639                 atomic_subtract_##name(ptr, val);                       \
 640         }
 641
 642 #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
 643         _ASAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 644         _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
 645         _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
 646
 647 #define _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 648         void kasan_atomic_set_##name(volatile type *ptr, type val)      \
 649         {                                                               \
 650                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 651                     __RET_ADDR);                                        \
 652                 atomic_set_##name(ptr, val);                            \
 653         }
 654
 655 #define ASAN_ATOMIC_FUNC_SET(name, type)                                \
 656         _ASAN_ATOMIC_FUNC_SET(name, type)                               \
 657         _ASAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
 658         _ASAN_ATOMIC_FUNC_SET(rel_##name, type)
 659
 660 #define _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 661         void kasan_atomic_clear_##name(volatile type *ptr, type val)    \
 662         {                                                               \
 663                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 664                     __RET_ADDR);                                        \
 665                 atomic_clear_##name(ptr, val);                          \
 666         }
 667
 668 #define ASAN_ATOMIC_FUNC_CLEAR(name, type)                              \
 669         _ASAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 670         _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
 671         _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
 672
 673 #define ASAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
 674         type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \
 675         {                                                               \
 676                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 677                     __RET_ADDR);                                        \
 678                 return (atomic_fetchadd_##name(ptr, val));              \
 679         }
 680
 681 #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
 682         type kasan_atomic_readandclear_##name(volatile type *ptr)       \
 683         {                                                               \
 684                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 685                     __RET_ADDR);                                        \
 686                 return (atomic_readandclear_##name(ptr));               \
 687         }
 688
 689 #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
 690         int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
 691         {                                                               \
 692                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 693                     __RET_ADDR);                                        \
 694                 return (atomic_testandclear_##name(ptr, v));            \
 695         }
 696
 697 #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
 698         int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
 699         {                                                               \
 700                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 701                     __RET_ADDR);                                        \
 702                 return (atomic_testandset_##name(ptr, v));              \
 703         }
 704
 705 #define ASAN_ATOMIC_FUNC_SWAP(name, type)                               \
 706         type kasan_atomic_swap_##name(volatile type *ptr, type val)     \
 707         {                                                               \
 708                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 709                     __RET_ADDR);                                        \
 710                 return (atomic_swap_##name(ptr, val));                  \
 711         }
 712
 713 #define _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 714         int kasan_atomic_cmpset_##name(volatile type *ptr, type oval,   \
 715             type nval)                                                  \
 716         {                                                               \
 717                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 718                     __RET_ADDR);                                        \
 719                 return (atomic_cmpset_##name(ptr, oval, nval));         \
 720         }
 721
 722 #define ASAN_ATOMIC_FUNC_CMPSET(name, type)                             \
 723         _ASAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 724         _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
 725         _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
 726
 727 #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 728         int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
 729             type nval)                                                  \
 730         {                                                               \
 731                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 732                     __RET_ADDR);                                        \
 733                 return (atomic_fcmpset_##name(ptr, oval, nval));        \
 734         }
 735
 736 #define ASAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
 737         _ASAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 738         _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
 739         _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
 740
 741 #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
 742         void kasan_atomic_thread_fence_##name(void)                     \
 743         {                                                               \
 744                 atomic_thread_fence_##name();                           \
 745         }
 746
 747 #define _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 748         type kasan_atomic_load_##name(volatile type *ptr)               \
 749         {                                                               \
 750                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 751                     __RET_ADDR);                                        \
 752                 return (atomic_load_##name(ptr));                       \
 753         }
 754
 755 #define ASAN_ATOMIC_FUNC_LOAD(name, type)                               \
 756         _ASAN_ATOMIC_FUNC_LOAD(name, type)                              \
 757         _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
 758
 759 #define _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 760         void kasan_atomic_store_##name(volatile type *ptr, type val)    \
 761         {                                                               \
 762                 kasan_shadow_check((uintptr_t)ptr, sizeof(type), true,  \
 763                     __RET_ADDR);                                        \
 764                 atomic_store_##name(ptr, val);                          \
 765         }
 766
 767 #define ASAN_ATOMIC_FUNC_STORE(name, type)                              \
 768         _ASAN_ATOMIC_FUNC_STORE(name, type)                             \
 769         _ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
 770
 771 ASAN_ATOMIC_FUNC_ADD(8, uint8_t);
 772 ASAN_ATOMIC_FUNC_ADD(16, uint16_t);
 773 ASAN_ATOMIC_FUNC_ADD(32, uint32_t);
 774 ASAN_ATOMIC_FUNC_ADD(64, uint64_t);
 775 ASAN_ATOMIC_FUNC_ADD(int, u_int);
 776 ASAN_ATOMIC_FUNC_ADD(long, u_long);
 777 ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
 778
 779 ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
 780 ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
 781 ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
 782 ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
 783 ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
 784 ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
 785 ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
 786
 787 ASAN_ATOMIC_FUNC_SET(8, uint8_t);
 788 ASAN_ATOMIC_FUNC_SET(16, uint16_t);
 789 ASAN_ATOMIC_FUNC_SET(32, uint32_t);
 790 ASAN_ATOMIC_FUNC_SET(64, uint64_t);
 791 ASAN_ATOMIC_FUNC_SET(int, u_int);
 792 ASAN_ATOMIC_FUNC_SET(long, u_long);
 793 ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
 794
 795 ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
 796 ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
 797 ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
 798 ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
 799 ASAN_ATOMIC_FUNC_CLEAR(int, u_int);
 800 ASAN_ATOMIC_FUNC_CLEAR(long, u_long);
 801 ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
 802
 803 ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
 804 ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
 805 ASAN_ATOMIC_FUNC_FETCHADD(int, u_int);
 806 ASAN_ATOMIC_FUNC_FETCHADD(long, u_long);
 807
 808 ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
 809 ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
 810 ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
 811 ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
 812 ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
 813
 814 ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
 815 ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
 816 ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
 817 ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
 818
 819 ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
 820 ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
 821 ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
 822 ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
 823
 824 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t);
 825 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t);
 826 ASAN_ATOMIC_FUNC_SWAP(int, u_int);
 827 ASAN_ATOMIC_FUNC_SWAP(long, u_long);
 828 ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
 829
 830 ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
 831 ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
 832 ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
 833 ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
 834 ASAN_ATOMIC_FUNC_CMPSET(int, u_int);
 835 ASAN_ATOMIC_FUNC_CMPSET(long, u_long);
 836 ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
 837
 838 ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
 839 ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
 840 ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
 841 ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
 842 ASAN_ATOMIC_FUNC_FCMPSET(int, u_int);
 843 ASAN_ATOMIC_FUNC_FCMPSET(long, u_long);
 844 ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
 845
 846 _ASAN_ATOMIC_FUNC_LOAD(bool, bool);
 847 ASAN_ATOMIC_FUNC_LOAD(8, uint8_t);
 848 ASAN_ATOMIC_FUNC_LOAD(16, uint16_t);
 849 ASAN_ATOMIC_FUNC_LOAD(32, uint32_t);
 850 ASAN_ATOMIC_FUNC_LOAD(64, uint64_t);
 851 ASAN_ATOMIC_FUNC_LOAD(char, u_char);
 852 ASAN_ATOMIC_FUNC_LOAD(short, u_short);
 853 ASAN_ATOMIC_FUNC_LOAD(int, u_int);
 854 ASAN_ATOMIC_FUNC_LOAD(long, u_long);
 855 ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
 856
 857 _ASAN_ATOMIC_FUNC_STORE(bool, bool);
 858 ASAN_ATOMIC_FUNC_STORE(8, uint8_t);
 859 ASAN_ATOMIC_FUNC_STORE(16, uint16_t);
 860 ASAN_ATOMIC_FUNC_STORE(32, uint32_t);
 861 ASAN_ATOMIC_FUNC_STORE(64, uint64_t);
 862 ASAN_ATOMIC_FUNC_STORE(char, u_char);
 863 ASAN_ATOMIC_FUNC_STORE(short, u_short);
 864 ASAN_ATOMIC_FUNC_STORE(int, u_int);
 865 ASAN_ATOMIC_FUNC_STORE(long, u_long);
 866 ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
 867
 868 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq);
 869 ASAN_ATOMIC_FUNC_THREAD_FENCE(rel);
 870 ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
 871 ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
 872
 873 void
 874 kasan_atomic_interrupt_fence(void)
 875 {
 876 }
 877
 878 /* -------------------------------------------------------------------------- */
 879
 880 #include <sys/bus.h>
 881 #include <machine/bus.h>
 882 #include <sys/bus_san.h>
 883
 884 int
 885 kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
 886     int flags, bus_space_handle_t *handlep)
 887 {
 888         return (bus_space_map(tag, hnd, size, flags, handlep));
 889 }
 890
 891 void
 892 kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
 893     bus_size_t size)
 894 {
 895         bus_space_unmap(tag, hnd, size);
 896 }
 897
 898 int
 899 kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
 900     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
 901 {
 902         return (bus_space_subregion(tag, hnd, offset, size, handlep));
 903 }
 904
 905 void
 906 kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
 907     bus_size_t size)
 908 {
 909         bus_space_free(tag, hnd, size);
 910 }
 911
 912 void
 913 kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
 914     bus_size_t offset, bus_size_t size, int flags)
 915 {
 916         bus_space_barrier(tag, hnd, offset, size, flags);
 917 }
 918
 919 #define ASAN_BUS_READ_FUNC(func, width, type)                           \
 920         type kasan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
 921             bus_space_handle_t hnd, bus_size_t offset)                  \
 922         {                                                               \
 923                 return (bus_space_read##func##_##width(tag, hnd,        \
 924                     offset));                                           \
 925         }                                                               \
 926
 927 #define ASAN_BUS_READ_PTR_FUNC(func, width, type)                       \
 928         void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
 929             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
 930             bus_size_t count)                                           \
 931         {                                                               \
 932                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 933                     false, __RET_ADDR);                                 \
 934                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
 935                     count);                                             \
 936         }
 937
 938 ASAN_BUS_READ_FUNC(, 1, uint8_t)
 939 ASAN_BUS_READ_FUNC(_stream, 1, uint8_t)
 940 ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
 941 ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
 942 ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
 943 ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
 944
 945 ASAN_BUS_READ_FUNC(, 2, uint16_t)
 946 ASAN_BUS_READ_FUNC(_stream, 2, uint16_t)
 947 ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
 948 ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
 949 ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
 950 ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
 951
 952 ASAN_BUS_READ_FUNC(, 4, uint32_t)
 953 ASAN_BUS_READ_FUNC(_stream, 4, uint32_t)
 954 ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
 955 ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
 956 ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
 957 ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
 958
 959 ASAN_BUS_READ_FUNC(, 8, uint64_t)
 960 #if defined(__aarch64__)
 961 ASAN_BUS_READ_FUNC(_stream, 8, uint64_t)
 962 ASAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
 963 ASAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
 964 ASAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
 965 ASAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
 966 #endif
 967
 968 #define ASAN_BUS_WRITE_FUNC(func, width, type)                          \
 969         void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \
 970             bus_space_handle_t hnd, bus_size_t offset, type value)      \
 971         {                                                               \
 972                 bus_space_write##func##_##width(tag, hnd, offset, value);\
 973         }                                                               \
 974
 975 #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
 976         void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
 977             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
 978             bus_size_t count)                                           \
 979         {                                                               \
 980                 kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
 981                     true, __RET_ADDR);                                  \
 982                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
 983                     count);                                             \
 984         }
 985
 986 ASAN_BUS_WRITE_FUNC(, 1, uint8_t)
 987 ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
 988 ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
 989 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
 990 ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
 991 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
 992
 993 ASAN_BUS_WRITE_FUNC(, 2, uint16_t)
 994 ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
 995 ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
 996 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
 997 ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
 998 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
 999
1000 ASAN_BUS_WRITE_FUNC(, 4, uint32_t)
1001 ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1002 ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1003 ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1004 ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1005 ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1006
1007 ASAN_BUS_WRITE_FUNC(, 8, uint64_t)
1008
1009 #define ASAN_BUS_SET_FUNC(func, width, type)                            \
1010         void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
1011             bus_space_handle_t hnd, bus_size_t offset, type value,      \
1012             bus_size_t count)                                           \
1013         {                                                               \
1014                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1015                     count);                                             \
1016         }
1017
1018 ASAN_BUS_SET_FUNC(multi, 1, uint8_t)
1019 ASAN_BUS_SET_FUNC(region, 1, uint8_t)
1020 ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1021 ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1022
1023 ASAN_BUS_SET_FUNC(multi, 2, uint16_t)
1024 ASAN_BUS_SET_FUNC(region, 2, uint16_t)
1025 ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1026 ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1027
1028 ASAN_BUS_SET_FUNC(multi, 4, uint32_t)
1029 ASAN_BUS_SET_FUNC(region, 4, uint32_t)
1030 ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1031 ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1032
1033 #define ASAN_BUS_PEEK_FUNC(width, type)                                 \
1034         int kasan_bus_space_peek_##width(bus_space_tag_t tag,           \
1035             bus_space_handle_t hnd, bus_size_t offset, type *valuep)    \
1036         {                                                               \
1037                 return (bus_space_peek_##width(tag, hnd, offset,        \
1038                     valuep));                                           \
1039         }
1040
1041 ASAN_BUS_PEEK_FUNC(1, uint8_t)
1042 ASAN_BUS_PEEK_FUNC(2, uint16_t)
1043 ASAN_BUS_PEEK_FUNC(4, uint32_t)
1044 ASAN_BUS_PEEK_FUNC(8, uint64_t)
1045
1046 #define ASAN_BUS_POKE_FUNC(width, type)                                 \
1047         int kasan_bus_space_poke_##width(bus_space_tag_t tag,           \
1048             bus_space_handle_t hnd, bus_size_t offset, type value)      \
1049         {                                                               \
1050                 return (bus_space_poke_##width(tag, hnd, offset,        \
1051                     value));                                            \
1052         }
1053
1054 ASAN_BUS_POKE_FUNC(1, uint8_t)
1055 ASAN_BUS_POKE_FUNC(2, uint16_t)
1056 ASAN_BUS_POKE_FUNC(4, uint32_t)
1057 ASAN_BUS_POKE_FUNC(8, uint64_t)
1058
1059 /* -------------------------------------------------------------------------- */
1060
1061 void __asan_register_globals(struct __asan_global *, size_t);
1062 void __asan_unregister_globals(struct __asan_global *, size_t);
1063
1064 void
1065 __asan_register_globals(struct __asan_global *globals, size_t n)
1066 {
1067         size_t i;
1068
1069         for (i = 0; i < n; i++) {
1070                 kasan_mark(globals[i].beg, globals[i].size,
1071                     globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
1072         }
1073 }
1074
1075 void
1076 __asan_unregister_globals(struct __asan_global *globals, size_t n)
1077 {
1078         size_t i;
1079
1080         for (i = 0; i < n; i++) {
1081                 kasan_mark(globals[i].beg, globals[i].size_with_redzone,
1082                     globals[i].size_with_redzone, 0);
1083         }
1084 }
1085
1086 #define ASAN_LOAD_STORE(size)                                   \
1087         void __asan_load##size(unsigned long);                  \
1088         void __asan_load##size(unsigned long addr)              \
1089         {                                                       \
1090                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1091         }                                                       \
1092         void __asan_load##size##_noabort(unsigned long);        \
1093         void __asan_load##size##_noabort(unsigned long addr)    \
1094         {                                                       \
1095                 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1096         }                                                       \
1097         void __asan_store##size(unsigned long);                 \
1098         void __asan_store##size(unsigned long addr)             \
1099         {                                                       \
1100                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1101         }                                                       \
1102         void __asan_store##size##_noabort(unsigned long);       \
1103         void __asan_store##size##_noabort(unsigned long addr)   \
1104         {                                                       \
1105                 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1106         }
1107
1108 ASAN_LOAD_STORE(1);
1109 ASAN_LOAD_STORE(2);
1110 ASAN_LOAD_STORE(4);
1111 ASAN_LOAD_STORE(8);
1112 ASAN_LOAD_STORE(16);
1113
1114 void __asan_loadN(unsigned long, size_t);
1115 void __asan_loadN_noabort(unsigned long, size_t);
1116 void __asan_storeN(unsigned long, size_t);
1117 void __asan_storeN_noabort(unsigned long, size_t);
1118 void __asan_handle_no_return(void);
1119
1120 void
1121 __asan_loadN(unsigned long addr, size_t size)
1122 {
1123         kasan_shadow_check(addr, size, false, __RET_ADDR);
1124 }
1125
1126 void
1127 __asan_loadN_noabort(unsigned long addr, size_t size)
1128 {
1129         kasan_shadow_check(addr, size, false, __RET_ADDR);
1130 }
1131
1132 void
1133 __asan_storeN(unsigned long addr, size_t size)
1134 {
1135         kasan_shadow_check(addr, size, true, __RET_ADDR);
1136 }
1137
1138 void
1139 __asan_storeN_noabort(unsigned long addr, size_t size)
1140 {
1141         kasan_shadow_check(addr, size, true, __RET_ADDR);
1142 }
1143
1144 void
1145 __asan_handle_no_return(void)
1146 {
1147         /* nothing */
1148 }
1149
1150 #define ASAN_SET_SHADOW(byte) \
1151         void __asan_set_shadow_##byte(void *, size_t);                  \
1152         void __asan_set_shadow_##byte(void *addr, size_t size)          \
1153         {                                                               \
1154                 __builtin_memset((void *)addr, 0x##byte, size);         \
1155         }
1156
1157 ASAN_SET_SHADOW(00);
1158 ASAN_SET_SHADOW(f1);
1159 ASAN_SET_SHADOW(f2);
1160 ASAN_SET_SHADOW(f3);
1161 ASAN_SET_SHADOW(f5);
1162 ASAN_SET_SHADOW(f8);
1163
1164 void __asan_poison_stack_memory(const void *, size_t);
1165 void __asan_unpoison_stack_memory(const void *, size_t);
1166
1167 void
1168 __asan_poison_stack_memory(const void *addr, size_t size)
1169 {
1170         size = roundup(size, KASAN_SHADOW_SCALE);
1171         kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1172 }
1173
1174 void
1175 __asan_unpoison_stack_memory(const void *addr, size_t size)
1176 {
1177         kasan_shadow_Nbyte_markvalid(addr, size);
1178 }
1179
1180 void __asan_alloca_poison(const void *, size_t);
1181 void __asan_allocas_unpoison(const void *, const void *);
1182
1183 void
1184 __asan_alloca_poison(const void *addr, size_t size)
1185 {
1186         const void *l, *r;
1187
1188         KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0,
1189             ("%s: invalid address %p", __func__, addr));
1190
1191         l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1192         r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1193
1194         kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1195         kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1196             KASAN_STACK_MID);
1197         kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1198 }
1199
1200 void
1201 __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1202 {
1203         size_t size;
1204
1205         if (__predict_false(!stkbegin))
1206                 return;
1207         if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1208                 return;
1209         size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1210
1211         kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1212 }
1213
1214 void __asan_poison_memory_region(const void *addr, size_t size);
1215 void __asan_unpoison_memory_region(const void *addr, size_t size);
1216
1217 void
1218 __asan_poison_memory_region(const void *addr, size_t size)
1219 {
1220 }
1221
1222 void
1223 __asan_unpoison_memory_region(const void *addr, size_t size)
1224 {
1225 }