sys/kern/subr_msan.c

   1 /*      $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $      */
   2
   3 /*
   4  * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
   5  * All rights reserved.
   6  * Copyright (c) 2021 The FreeBSD Foundation
   7  *
   8  * Portions of this software were developed by Mark Johnston under sponsorship
   9  * from the FreeBSD Foundation.
  10  *
  11  * This code is part of the KMSAN subsystem of the NetBSD kernel.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  23  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  24  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  25  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 #define SAN_RUNTIME
  36
  37 #include <sys/cdefs.h>
  38 #if 0
  39 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $");
  40 #endif
  41
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/bio.h>
  45 #include <sys/buf.h>
  46 #include <sys/conf.h>
  47 #include <sys/kdb.h>
  48 #include <sys/kernel.h>
  49 #include <sys/linker.h>
  50 #include <sys/malloc.h>
  51 #include <sys/mbuf.h>
  52 #include <sys/memdesc.h>
  53 #include <sys/msan.h>
  54 #include <sys/proc.h>
  55 #include <sys/stack.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/uio.h>
  58
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61
  62 #include <machine/msan.h>
  63 #include <machine/stdarg.h>
  64
  65 void kmsan_init_arg(size_t);
  66 void kmsan_init_ret(size_t);
  67
  68 /* -------------------------------------------------------------------------- */
  69
  70 /*
  71  * Part of the compiler ABI.
  72  */
  73
  74 typedef struct {
  75         uint8_t *shad;
  76         msan_orig_t *orig;
  77 } msan_meta_t;
  78
  79 #define MSAN_PARAM_SIZE         800
  80 #define MSAN_RETVAL_SIZE        800
  81 typedef struct {
  82         uint8_t param_shadow[MSAN_PARAM_SIZE];
  83         uint8_t retval_shadow[MSAN_RETVAL_SIZE];
  84         uint8_t va_arg_shadow[MSAN_PARAM_SIZE];
  85         uint8_t va_arg_origin[MSAN_PARAM_SIZE];
  86         uint64_t va_arg_overflow_size;
  87         msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)];
  88         msan_orig_t retval_origin;
  89 } msan_tls_t;
  90
  91 /* -------------------------------------------------------------------------- */
  92
  93 #define MSAN_NCONTEXT   4
  94 #define MSAN_ORIG_MASK  (~0x3)
  95
  96 typedef struct kmsan_td {
  97         size_t ctx;
  98         msan_tls_t tls[MSAN_NCONTEXT];
  99 } msan_td_t;
 100
 101 static msan_tls_t dummy_tls;
 102
 103 /*
 104  * Use separate dummy regions for loads and stores: stores may mark the region
 105  * as uninitialized, and that can trigger false positives.
 106  */
 107 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
 108 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
 109 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
 110 static msan_td_t msan_thread0;
 111 static bool kmsan_reporting = false;
 112
 113 /*
 114  * Avoid clobbering any thread-local state before we panic.
 115  */
 116 #define kmsan_panic(f, ...) do {                        \
 117         kmsan_disabled = true;                          \
 118         panic(f, __VA_ARGS__);                          \
 119 } while (0)
 120
 121 #define REPORT(f, ...) do {                             \
 122         if (panic_on_violation) {                       \
 123                 kmsan_panic(f, __VA_ARGS__);            \
 124         } else {                                        \
 125                 struct stack st;                        \
 126                                                         \
 127                 stack_save(&st);                        \
 128                 printf(f "\n", __VA_ARGS__);            \
 129                 stack_print_ddb(&st);                   \
 130         }                                               \
 131 } while (0)
 132
 133 FEATURE(kmsan, "Kernel memory sanitizer");
 134
 135 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 136     "KMSAN options");
 137
 138 static bool panic_on_violation = 1;
 139 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN,
 140     &panic_on_violation, 0,
 141     "Panic if an invalid access is detected");
 142
 143 static bool kmsan_disabled __read_mostly = true;
 144 #define kmsan_enabled (!kmsan_disabled)
 145 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
 146     &kmsan_disabled, 0, "KMSAN is disabled");
 147
 148 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer");
 149
 150 /* -------------------------------------------------------------------------- */
 151
 152 static inline const char *
 153 kmsan_orig_name(int type)
 154 {
 155         switch (type) {
 156         case KMSAN_TYPE_STACK:
 157                 return ("stack");
 158         case KMSAN_TYPE_KMEM:
 159                 return ("kmem");
 160         case KMSAN_TYPE_MALLOC:
 161                 return ("malloc");
 162         case KMSAN_TYPE_UMA:
 163                 return ("UMA");
 164         default:
 165                 return ("unknown");
 166         }
 167 }
 168
 169 static void
 170 kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off,
 171     const char *hook)
 172 {
 173         const char *typename;
 174         char *var, *fn;
 175         uintptr_t ptr;
 176         long foff;
 177         char buf[128];
 178         int type;
 179
 180         if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
 181                 return;
 182
 183         kmsan_reporting = true;
 184         __compiler_membar();
 185
 186         if (*orig == 0) {
 187                 REPORT("MSan: Uninitialized memory in %s, offset %zu",
 188                     hook, off);
 189                 goto out;
 190         }
 191
 192         kmsan_md_orig_decode(*orig, &type, &ptr);
 193         typename = kmsan_orig_name(type);
 194
 195         if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
 196             sizeof(buf), &foff) == 0) {
 197                 REPORT("MSan: Uninitialized %s memory in %s, "
 198                     "offset %zu/%zu, addr %p, from %s+%#lx",
 199                     typename, hook, off, size, addr, buf, foff);
 200         } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
 201                 /*
 202                  * The format of the string is: "----var@function". Parse it to
 203                  * display a nice warning.
 204                  */
 205                 var = (char *)ptr + 4;
 206                 strlcpy(buf, var, sizeof(buf));
 207                 var = buf;
 208                 fn = strchr(buf, '@');
 209                 *fn++ = '\0';
 210                 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, "
 211                     "variable '%s' from %s", typename, hook, off, var, fn);
 212         } else {
 213                 REPORT("MSan: Uninitialized %s memory in %s, "
 214                     "offset %zu/%zu, addr %p, PC %p",
 215                     typename, hook, off, size, addr, (void *)ptr);
 216         }
 217
 218 out:
 219         __compiler_membar();
 220         kmsan_reporting = false;
 221 }
 222
 223 static void
 224 kmsan_report_inline(msan_orig_t orig, unsigned long pc)
 225 {
 226         const char *typename;
 227         char *var, *fn;
 228         uintptr_t ptr;
 229         char buf[128];
 230         long foff;
 231         int type;
 232
 233         if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
 234                 return;
 235
 236         kmsan_reporting = true;
 237         __compiler_membar();
 238
 239         if (orig == 0) {
 240                 REPORT("MSan: uninitialized variable in %p", (void *)pc);
 241                 goto out;
 242         }
 243
 244         kmsan_md_orig_decode(orig, &type, &ptr);
 245         typename = kmsan_orig_name(type);
 246
 247         if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
 248             sizeof(buf), &foff) == 0) {
 249                 REPORT("MSan: Uninitialized %s memory from %s+%#lx",
 250                     typename, buf, foff);
 251         } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
 252                 /*
 253                  * The format of the string is: "----var@function". Parse it to
 254                  * display a nice warning.
 255                  */
 256                 var = (char *)ptr + 4;
 257                 strlcpy(buf, var, sizeof(buf));
 258                 var = buf;
 259                 fn = strchr(buf, '@');
 260                 *fn++ = '\0';
 261                 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn);
 262         } else {
 263                 REPORT("MSan: Uninitialized %s memory, origin %x",
 264                     typename, orig);
 265         }
 266
 267 out:
 268         __compiler_membar();
 269         kmsan_reporting = false;
 270 }
 271
 272 /* -------------------------------------------------------------------------- */
 273
 274 static inline msan_meta_t
 275 kmsan_meta_get(const void *addr, size_t size, const bool write)
 276 {
 277         msan_meta_t ret;
 278
 279         if (__predict_false(!kmsan_enabled)) {
 280                 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
 281                 ret.orig = (msan_orig_t *)msan_dummy_orig;
 282         } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) {
 283                 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
 284                 ret.orig = (msan_orig_t *)msan_dummy_orig;
 285         } else {
 286                 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr);
 287                 ret.orig =
 288                     (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
 289                 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig &
 290                     MSAN_ORIG_MASK);
 291         }
 292
 293         return (ret);
 294 }
 295
 296 static inline void
 297 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size)
 298 {
 299         msan_orig_t *orig;
 300         size_t i;
 301
 302         if (__predict_false(!kmsan_enabled))
 303                 return;
 304         if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr)))
 305                 return;
 306
 307         orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
 308         size += ((uintptr_t)orig & (sizeof(*orig) - 1));
 309         orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
 310
 311         for (i = 0; i < size; i += 4) {
 312                 orig[i / 4] = o;
 313         }
 314 }
 315
 316 static inline void
 317 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size)
 318 {
 319         uint8_t *shad;
 320
 321         if (__predict_false(!kmsan_enabled))
 322                 return;
 323         if (__predict_false(kmsan_md_unsupported(addr)))
 324                 return;
 325
 326         shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
 327         __builtin_memset(shad, c, size);
 328 }
 329
 330 static inline void
 331 kmsan_meta_copy(void *dst, const void *src, size_t size)
 332 {
 333         uint8_t *orig_src, *orig_dst;
 334         uint8_t *shad_src, *shad_dst;
 335         msan_orig_t *_src, *_dst;
 336         size_t i;
 337
 338         if (__predict_false(!kmsan_enabled))
 339                 return;
 340         if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst)))
 341                 return;
 342         if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) {
 343                 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size);
 344                 return;
 345         }
 346
 347         shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src);
 348         shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst);
 349         __builtin_memmove(shad_dst, shad_src, size);
 350
 351         orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src);
 352         orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst);
 353         for (i = 0; i < size; i++) {
 354                 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK);
 355                 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK);
 356                 *_dst = *_src;
 357                 orig_src++;
 358                 orig_dst++;
 359         }
 360 }
 361
 362 static inline void
 363 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook)
 364 {
 365         msan_orig_t *orig;
 366         uint8_t *shad;
 367         size_t i;
 368
 369         if (__predict_false(!kmsan_enabled))
 370                 return;
 371         if (__predict_false(kmsan_md_unsupported(addr)))
 372                 return;
 373
 374         shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
 375         for (i = 0; i < size; i++) {
 376                 if (__predict_true(shad[i] == 0))
 377                         continue;
 378                 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr + i);
 379                 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
 380                 kmsan_report_hook((const char *)addr + i, orig, size, i, hook);
 381                 break;
 382         }
 383 }
 384
 385 void
 386 kmsan_init_arg(size_t n)
 387 {
 388         msan_td_t *mtd;
 389         uint8_t *arg;
 390
 391         if (__predict_false(!kmsan_enabled))
 392                 return;
 393         if (__predict_false(curthread == NULL))
 394                 return;
 395         mtd = curthread->td_kmsan;
 396         arg = mtd->tls[mtd->ctx].param_shadow;
 397         __builtin_memset(arg, 0, n);
 398 }
 399
 400 void
 401 kmsan_init_ret(size_t n)
 402 {
 403         msan_td_t *mtd;
 404         uint8_t *arg;
 405
 406         if (__predict_false(!kmsan_enabled))
 407                 return;
 408         if (__predict_false(curthread == NULL))
 409                 return;
 410         mtd = curthread->td_kmsan;
 411         arg = mtd->tls[mtd->ctx].retval_shadow;
 412         __builtin_memset(arg, 0, n);
 413 }
 414
 415 static void
 416 kmsan_check_arg(size_t size, const char *hook)
 417 {
 418         msan_orig_t *orig;
 419         msan_td_t *mtd;
 420         uint8_t *arg;
 421         size_t ctx, i;
 422
 423         if (__predict_false(!kmsan_enabled))
 424                 return;
 425         if (__predict_false(curthread == NULL))
 426                 return;
 427         mtd = curthread->td_kmsan;
 428         ctx = mtd->ctx;
 429         arg = mtd->tls[ctx].param_shadow;
 430
 431         for (i = 0; i < size; i++) {
 432                 if (__predict_true(arg[i] == 0))
 433                         continue;
 434                 orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)];
 435                 kmsan_report_hook((const char *)arg + i, orig, size, i, hook);
 436                 break;
 437         }
 438 }
 439
 440 void
 441 kmsan_thread_alloc(struct thread *td)
 442 {
 443         msan_td_t *mtd;
 444
 445         if (!kmsan_enabled)
 446                 return;
 447
 448         mtd = td->td_kmsan;
 449         if (mtd == NULL) {
 450                 /* We might be recycling a thread. */
 451                 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) +
 452                     sizeof(int));
 453                 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK);
 454         }
 455         __builtin_memset(mtd, 0, sizeof(*mtd));
 456         mtd->ctx = 0;
 457
 458         if (td->td_kstack != 0)
 459                 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages),
 460                     KMSAN_STATE_UNINIT);
 461
 462         td->td_kmsan = mtd;
 463 }
 464
 465 void
 466 kmsan_thread_free(struct thread *td)
 467 {
 468         msan_td_t *mtd;
 469
 470         if (!kmsan_enabled)
 471                 return;
 472         if (__predict_false(td == curthread))
 473                 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__);
 474
 475         mtd = td->td_kmsan;
 476         kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *));
 477         free(mtd, M_KMSAN);
 478         td->td_kmsan = NULL;
 479 }
 480
 481 void kmsan_intr_enter(void);
 482 void kmsan_intr_leave(void);
 483
 484 void
 485 kmsan_intr_enter(void)
 486 {
 487         msan_td_t *mtd;
 488
 489         if (__predict_false(!kmsan_enabled))
 490                 return;
 491
 492         mtd = curthread->td_kmsan;
 493         mtd->ctx++;
 494         if (__predict_false(mtd->ctx >= MSAN_NCONTEXT))
 495                 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
 496 }
 497
 498 void
 499 kmsan_intr_leave(void)
 500 {
 501         msan_td_t *mtd;
 502
 503         if (__predict_false(!kmsan_enabled))
 504                 return;
 505
 506         mtd = curthread->td_kmsan;
 507         if (__predict_false(mtd->ctx == 0))
 508                 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
 509         mtd->ctx--;
 510 }
 511
 512 /* -------------------------------------------------------------------------- */
 513
 514 void
 515 kmsan_shadow_map(vm_offset_t addr, size_t size)
 516 {
 517         size_t npages, i;
 518         vm_offset_t va;
 519
 520         MPASS(addr % PAGE_SIZE == 0);
 521         MPASS(size % PAGE_SIZE == 0);
 522
 523         if (!kmsan_enabled)
 524                 return;
 525
 526         npages = atop(size);
 527
 528         va = kmsan_md_addr_to_shad(addr);
 529         for (i = 0; i < npages; i++) {
 530                 pmap_san_enter(va + ptoa(i));
 531         }
 532
 533         va = kmsan_md_addr_to_orig(addr);
 534         for (i = 0; i < npages; i++) {
 535                 pmap_san_enter(va + ptoa(i));
 536         }
 537 }
 538
 539 void
 540 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc)
 541 {
 542         msan_orig_t orig;
 543
 544         orig = kmsan_md_orig_encode(type, pc);
 545         kmsan_origin_fill(addr, orig, size);
 546 }
 547
 548 void
 549 kmsan_mark(const void *addr, size_t size, uint8_t c)
 550 {
 551         kmsan_shadow_fill((uintptr_t)addr, c, size);
 552 }
 553
 554 void
 555 kmsan_mark_bio(const struct bio *bp, uint8_t c)
 556 {
 557         kmsan_mark(bp->bio_data, bp->bio_length, c);
 558 }
 559
 560 void
 561 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c)
 562 {
 563         do {
 564                 if ((m->m_flags & M_EXTPG) == 0)
 565                         kmsan_mark(m->m_data, m->m_len, c);
 566                 m = m->m_next;
 567         } while (m != NULL);
 568 }
 569
 570 void
 571 kmsan_check(const void *p, size_t sz, const char *descr)
 572 {
 573         kmsan_shadow_check((uintptr_t)p, sz, descr);
 574 }
 575
 576 void
 577 kmsan_check_bio(const struct bio *bp, const char *descr)
 578 {
 579         kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr);
 580 }
 581
 582 void
 583 kmsan_check_mbuf(const struct mbuf *m, const char *descr)
 584 {
 585         do {
 586                 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr);
 587         } while ((m = m->m_next) != NULL);
 588 }
 589
 590 void
 591 kmsan_check_uio(const struct uio *uio, const char *descr)
 592 {
 593         for (int i = 0; i < uio->uio_iovcnt; i++) {
 594                 kmsan_check(uio->uio_iov[i].iov_base, uio->uio_iov[i].iov_len,
 595                     descr);
 596         }
 597 }
 598
 599 void
 600 kmsan_init(void)
 601 {
 602         int disabled;
 603
 604         disabled = 0;
 605         TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled);
 606         if (disabled)
 607                 return;
 608
 609         /* Initialize the TLS for curthread. */
 610         msan_thread0.ctx = 0;
 611         thread0.td_kmsan = &msan_thread0;
 612
 613         /* Now officially enabled. */
 614         kmsan_disabled = false;
 615 }
 616
 617 /* -------------------------------------------------------------------------- */
 618
 619 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
 620 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
 621
 622 msan_meta_t
 623 __msan_metadata_ptr_for_load_n(void *addr, size_t size)
 624 {
 625         return (kmsan_meta_get(addr, size, false));
 626 }
 627
 628 msan_meta_t
 629 __msan_metadata_ptr_for_store_n(void *addr, size_t size)
 630 {
 631         return (kmsan_meta_get(addr, size, true));
 632 }
 633
 634 #define MSAN_META_FUNC(size)                                            \
 635         msan_meta_t __msan_metadata_ptr_for_load_##size(void *);        \
 636         msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr)     \
 637         {                                                               \
 638                 return (kmsan_meta_get(addr, size, false));             \
 639         }                                                               \
 640         msan_meta_t __msan_metadata_ptr_for_store_##size(void *);       \
 641         msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr)    \
 642         {                                                               \
 643                 return (kmsan_meta_get(addr, size, true));              \
 644         }
 645
 646 MSAN_META_FUNC(1)
 647 MSAN_META_FUNC(2)
 648 MSAN_META_FUNC(4)
 649 MSAN_META_FUNC(8)
 650
 651 void __msan_instrument_asm_store(const void *, size_t);
 652 msan_orig_t __msan_chain_origin(msan_orig_t);
 653 void __msan_poison(const void *, size_t);
 654 void __msan_unpoison(const void *, size_t);
 655 void __msan_poison_alloca(const void *, uint64_t, const char *);
 656 void __msan_unpoison_alloca(const void *, uint64_t);
 657 void __msan_warning(msan_orig_t);
 658 msan_tls_t *__msan_get_context_state(void);
 659
 660 void
 661 __msan_instrument_asm_store(const void *addr, size_t size)
 662 {
 663         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
 664 }
 665
 666 msan_orig_t
 667 __msan_chain_origin(msan_orig_t origin)
 668 {
 669         return (origin);
 670 }
 671
 672 void
 673 __msan_poison(const void *addr, size_t size)
 674 {
 675         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
 676 }
 677
 678 void
 679 __msan_unpoison(const void *addr, size_t size)
 680 {
 681         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
 682 }
 683
 684 void
 685 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr)
 686 {
 687         msan_orig_t orig;
 688
 689         orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
 690         kmsan_origin_fill(addr, orig, size);
 691         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
 692 }
 693
 694 void
 695 __msan_unpoison_alloca(const void *addr, uint64_t size)
 696 {
 697         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
 698 }
 699
 700 void
 701 __msan_warning(msan_orig_t origin)
 702 {
 703         if (__predict_false(!kmsan_enabled))
 704                 return;
 705         kmsan_report_inline(origin, KMSAN_RET_ADDR);
 706 }
 707
 708 msan_tls_t *
 709 __msan_get_context_state(void)
 710 {
 711         msan_td_t *mtd;
 712
 713         /*
 714          * When APs are started, they execute some C code before curthread is
 715          * set.  We have to handle that here.
 716          */
 717         if (__predict_false(!kmsan_enabled || curthread == NULL))
 718                 return (&dummy_tls);
 719         mtd = curthread->td_kmsan;
 720         return (&mtd->tls[mtd->ctx]);
 721 }
 722
 723 /* -------------------------------------------------------------------------- */
 724
 725 /*
 726  * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
 727  * initialized areas properly.
 728  */
 729
 730 void *
 731 kmsan_memcpy(void *dst, const void *src, size_t len)
 732 {
 733         /* No kmsan_check_arg, because inlined. */
 734         kmsan_init_ret(sizeof(void *));
 735         if (__predict_true(len != 0)) {
 736                 kmsan_meta_copy(dst, src, len);
 737         }
 738         return (__builtin_memcpy(dst, src, len));
 739 }
 740
 741 int
 742 kmsan_memcmp(const void *b1, const void *b2, size_t len)
 743 {
 744         const uint8_t *_b1 = b1, *_b2 = b2;
 745         size_t i;
 746
 747         kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len),
 748             "memcmp():args");
 749         kmsan_init_ret(sizeof(int));
 750
 751         for (i = 0; i < len; i++) {
 752                 if (*_b1 != *_b2) {
 753                         kmsan_shadow_check((uintptr_t)b1, i + 1,
 754                             "memcmp():arg1");
 755                         kmsan_shadow_check((uintptr_t)b2, i + 1,
 756                             "memcmp():arg2");
 757                         return (*_b1 - *_b2);
 758                 }
 759                 _b1++, _b2++;
 760         }
 761
 762         return (0);
 763 }
 764
 765 void *
 766 kmsan_memset(void *dst, int c, size_t len)
 767 {
 768         /* No kmsan_check_arg, because inlined. */
 769         kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len);
 770         kmsan_init_ret(sizeof(void *));
 771         return (__builtin_memset(dst, c, len));
 772 }
 773
 774 void *
 775 kmsan_memmove(void *dst, const void *src, size_t len)
 776 {
 777         /* No kmsan_check_arg, because inlined. */
 778         if (__predict_true(len != 0)) {
 779                 kmsan_meta_copy(dst, src, len);
 780         }
 781         kmsan_init_ret(sizeof(void *));
 782         return (__builtin_memmove(dst, src, len));
 783 }
 784
 785 __strong_reference(kmsan_memcpy, __msan_memcpy);
 786 __strong_reference(kmsan_memset, __msan_memset);
 787 __strong_reference(kmsan_memmove, __msan_memmove);
 788
 789 char *
 790 kmsan_strcpy(char *dst, const char *src)
 791 {
 792         const char *_src = src;
 793         char *_dst = dst;
 794         size_t len = 0;
 795
 796         kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args");
 797
 798         while (1) {
 799                 len++;
 800                 *dst = *src;
 801                 if (*src == '\0')
 802                         break;
 803                 src++, dst++;
 804         }
 805
 806         kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2");
 807         kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len);
 808         kmsan_init_ret(sizeof(char *));
 809         return (_dst);
 810 }
 811
 812 int
 813 kmsan_strcmp(const char *s1, const char *s2)
 814 {
 815         const char *_s1 = s1, *_s2 = s2;
 816         size_t len = 0;
 817
 818         kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args");
 819         kmsan_init_ret(sizeof(int));
 820
 821         while (1) {
 822                 len++;
 823                 if (*s1 != *s2)
 824                         break;
 825                 if (*s1 == '\0') {
 826                         kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
 827                         kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
 828                         return (0);
 829                 }
 830                 s1++, s2++;
 831         }
 832
 833         kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
 834         kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
 835
 836         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 837 }
 838
 839 size_t
 840 kmsan_strlen(const char *str)
 841 {
 842         const char *s;
 843
 844         kmsan_check_arg(sizeof(str), "strlen():args");
 845
 846         s = str;
 847         while (1) {
 848                 if (*s == '\0')
 849                         break;
 850                 s++;
 851         }
 852
 853         kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1");
 854         kmsan_init_ret(sizeof(size_t));
 855         return (s - str);
 856 }
 857
 858 int     kmsan_copyin(const void *, void *, size_t);
 859 int     kmsan_copyout(const void *, void *, size_t);
 860 int     kmsan_copyinstr(const void *, void *, size_t, size_t *);
 861
 862 int
 863 kmsan_copyin(const void *uaddr, void *kaddr, size_t len)
 864 {
 865         int ret;
 866
 867         kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len),
 868             "copyin():args");
 869         ret = copyin(uaddr, kaddr, len);
 870         if (ret == 0)
 871                 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len);
 872         kmsan_init_ret(sizeof(int));
 873         return (ret);
 874 }
 875
 876 int
 877 kmsan_copyout(const void *kaddr, void *uaddr, size_t len)
 878 {
 879         kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len),
 880             "copyout():args");
 881         kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1");
 882         kmsan_init_ret(sizeof(int));
 883         return (copyout(kaddr, uaddr, len));
 884 }
 885
 886 int
 887 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 888 {
 889         size_t _done;
 890         int ret;
 891
 892         kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) +
 893             sizeof(len) + sizeof(done), "copyinstr():args");
 894         ret = copyinstr(uaddr, kaddr, len, &_done);
 895         if (ret == 0)
 896                 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done);
 897         if (done != NULL) {
 898                 *done = _done;
 899                 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t));
 900         }
 901         kmsan_init_ret(sizeof(int));
 902         return (ret);
 903 }
 904
 905 /* -------------------------------------------------------------------------- */
 906
 907 int
 908 kmsan_fubyte(volatile const void *base)
 909 {
 910         int ret;
 911
 912         kmsan_check_arg(sizeof(base), "fubyte(): args");
 913         ret = fubyte(base);
 914         kmsan_init_ret(sizeof(int));
 915         return (ret);
 916 }
 917
 918 int
 919 kmsan_fuword16(volatile const void *base)
 920 {
 921         int ret;
 922
 923         kmsan_check_arg(sizeof(base), "fuword16(): args");
 924         ret = fuword16(base);
 925         kmsan_init_ret(sizeof(int));
 926         return (ret);
 927 }
 928
 929 int
 930 kmsan_fueword(volatile const void *base, long *val)
 931 {
 932         int ret;
 933
 934         kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args");
 935         ret = fueword(base, val);
 936         if (ret == 0)
 937                 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
 938                     sizeof(*val));
 939         kmsan_init_ret(sizeof(int));
 940         return (ret);
 941 }
 942
 943 int
 944 kmsan_fueword32(volatile const void *base, int32_t *val)
 945 {
 946         int ret;
 947
 948         kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args");
 949         ret = fueword32(base, val);
 950         if (ret == 0)
 951                 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
 952                     sizeof(*val));
 953         kmsan_init_ret(sizeof(int));
 954         return (ret);
 955 }
 956
 957 int
 958 kmsan_fueword64(volatile const void *base, int64_t *val)
 959 {
 960         int ret;
 961
 962         kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args");
 963         ret = fueword64(base, val);
 964         if (ret == 0)
 965                 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
 966                     sizeof(*val));
 967         kmsan_init_ret(sizeof(int));
 968         return (ret);
 969 }
 970
 971 int
 972 kmsan_subyte(volatile void *base, int byte)
 973 {
 974         int ret;
 975
 976         kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args");
 977         ret = subyte(base, byte);
 978         kmsan_init_ret(sizeof(int));
 979         return (ret);
 980 }
 981
 982 int
 983 kmsan_suword(volatile void *base, long word)
 984 {
 985         int ret;
 986
 987         kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args");
 988         ret = suword(base, word);
 989         kmsan_init_ret(sizeof(int));
 990         return (ret);
 991 }
 992
 993 int
 994 kmsan_suword16(volatile void *base, int word)
 995 {
 996         int ret;
 997
 998         kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args");
 999         ret = suword16(base, word);
1000         kmsan_init_ret(sizeof(int));
1001         return (ret);
1002 }
1003
1004 int
1005 kmsan_suword32(volatile void *base, int32_t word)
1006 {
1007         int ret;
1008
1009         kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args");
1010         ret = suword32(base, word);
1011         kmsan_init_ret(sizeof(int));
1012         return (ret);
1013 }
1014
1015 int
1016 kmsan_suword64(volatile void *base, int64_t word)
1017 {
1018         int ret;
1019
1020         kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args");
1021         ret = suword64(base, word);
1022         kmsan_init_ret(sizeof(int));
1023         return (ret);
1024 }
1025
1026 int
1027 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
1028     uint32_t newval)
1029 {
1030         int ret;
1031
1032         kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1033             sizeof(newval), "casueword32(): args");
1034         ret = casueword32(base, oldval, oldvalp, newval);
1035         kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1036             sizeof(*oldvalp));
1037         kmsan_init_ret(sizeof(int));
1038         return (ret);
1039 }
1040
1041 int
1042 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
1043     u_long newval)
1044 {
1045         int ret;
1046
1047         kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1048             sizeof(newval), "casueword32(): args");
1049         ret = casueword(base, oldval, oldvalp, newval);
1050         kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1051             sizeof(*oldvalp));
1052         kmsan_init_ret(sizeof(int));
1053         return (ret);
1054 }
1055
1056 /* -------------------------------------------------------------------------- */
1057
1058 #include <machine/atomic.h>
1059 #include <sys/atomic_san.h>
1060
1061 #define _MSAN_ATOMIC_FUNC_ADD(name, type)                               \
1062         void kmsan_atomic_add_##name(volatile type *ptr, type val)      \
1063         {                                                               \
1064                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1065                     "atomic_add_" #name "():args");                     \
1066                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1067                     "atomic_add_" #name "():ptr");                      \
1068                 atomic_add_##name(ptr, val);                            \
1069         }
1070
1071 #define MSAN_ATOMIC_FUNC_ADD(name, type)                                \
1072         _MSAN_ATOMIC_FUNC_ADD(name, type)                               \
1073         _MSAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
1074         _MSAN_ATOMIC_FUNC_ADD(rel_##name, type)
1075
1076 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
1077         void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \
1078         {                                                               \
1079                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1080                     "atomic_subtract_" #name "():args");                \
1081                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1082                     "atomic_subtract_" #name "():ptr");                 \
1083                 atomic_subtract_##name(ptr, val);                       \
1084         }
1085
1086 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
1087         _MSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
1088         _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
1089         _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
1090
1091 #define _MSAN_ATOMIC_FUNC_SET(name, type)                               \
1092         void kmsan_atomic_set_##name(volatile type *ptr, type val)      \
1093         {                                                               \
1094                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1095                     "atomic_set_" #name "():args");                     \
1096                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1097                     "atomic_set_" #name "():ptr");                      \
1098                 atomic_set_##name(ptr, val);                            \
1099         }
1100
1101 #define MSAN_ATOMIC_FUNC_SET(name, type)                                \
1102         _MSAN_ATOMIC_FUNC_SET(name, type)                               \
1103         _MSAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
1104         _MSAN_ATOMIC_FUNC_SET(rel_##name, type)
1105
1106 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
1107         void kmsan_atomic_clear_##name(volatile type *ptr, type val)    \
1108         {                                                               \
1109                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1110                     "atomic_clear_" #name "():args");                   \
1111                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1112                     "atomic_clear_" #name "():ptr");                    \
1113                 atomic_clear_##name(ptr, val);                          \
1114         }
1115
1116 #define MSAN_ATOMIC_FUNC_CLEAR(name, type)                              \
1117         _MSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
1118         _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
1119         _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
1120
1121 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
1122         type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
1123         {                                                               \
1124                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1125                     "atomic_fetchadd_" #name "():args");                \
1126                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1127                     "atomic_fetchadd_" #name "():ptr");                 \
1128                 kmsan_init_ret(sizeof(type));                           \
1129                 return (atomic_fetchadd_##name(ptr, val));              \
1130         }
1131
1132 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
1133         type kmsan_atomic_readandclear_##name(volatile type *ptr)       \
1134         {                                                               \
1135                 kmsan_check_arg(sizeof(ptr),                            \
1136                     "atomic_readandclear_" #name "():args");            \
1137                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1138                     "atomic_readandclear_" #name "():ptr");             \
1139                 kmsan_init_ret(sizeof(type));                           \
1140                 return (atomic_readandclear_##name(ptr));               \
1141         }
1142
1143 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
1144         int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
1145         {                                                               \
1146                 kmsan_check_arg(sizeof(ptr) + sizeof(v),                \
1147                     "atomic_testandclear_" #name "():args");            \
1148                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1149                     "atomic_testandclear_" #name "():ptr");             \
1150                 kmsan_init_ret(sizeof(int));                            \
1151                 return (atomic_testandclear_##name(ptr, v));            \
1152         }
1153
1154 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
1155         int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \
1156         {                                                               \
1157                 kmsan_check_arg(sizeof(ptr) + sizeof(v),                \
1158                     "atomic_testandset_" #name "():args");              \
1159                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1160                     "atomic_testandset_" #name "():ptr");               \
1161                 kmsan_init_ret(sizeof(int));                            \
1162                 return (atomic_testandset_##name(ptr, v));              \
1163         }
1164
1165 #define MSAN_ATOMIC_FUNC_SWAP(name, type)                               \
1166         type kmsan_atomic_swap_##name(volatile type *ptr, type val)     \
1167         {                                                               \
1168                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1169                     "atomic_swap_" #name "():args");                    \
1170                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1171                     "atomic_swap_" #name "():ptr");                     \
1172                 kmsan_init_ret(sizeof(type));                           \
1173                 return (atomic_swap_##name(ptr, val));                  \
1174         }
1175
1176 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
1177         int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval,   \
1178             type nval)                                                  \
1179         {                                                               \
1180                 kmsan_check_arg(sizeof(ptr) + sizeof(oval) +            \
1181                     sizeof(nval), "atomic_cmpset_" #name "():args");    \
1182                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1183                     "atomic_cmpset_" #name "():ptr");                   \
1184                 kmsan_init_ret(sizeof(int));                            \
1185                 return (atomic_cmpset_##name(ptr, oval, nval));         \
1186         }
1187
1188 #define MSAN_ATOMIC_FUNC_CMPSET(name, type)                             \
1189         _MSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
1190         _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
1191         _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
1192
1193 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
1194         int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
1195             type nval)                                                  \
1196         {                                                               \
1197                 kmsan_check_arg(sizeof(ptr) + sizeof(oval) +            \
1198                     sizeof(nval), "atomic_fcmpset_" #name "():args");   \
1199                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1200                     "atomic_fcmpset_" #name "():ptr");                  \
1201                 kmsan_init_ret(sizeof(int));                            \
1202                 return (atomic_fcmpset_##name(ptr, oval, nval));        \
1203         }
1204
1205 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
1206         _MSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
1207         _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
1208         _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
1209
1210 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
1211         void kmsan_atomic_thread_fence_##name(void)                     \
1212         {                                                               \
1213                 atomic_thread_fence_##name();                           \
1214         }
1215
1216 #define _MSAN_ATOMIC_FUNC_LOAD(name, type)                              \
1217         type kmsan_atomic_load_##name(volatile type *ptr)               \
1218         {                                                               \
1219                 kmsan_check_arg(sizeof(ptr),                            \
1220                     "atomic_load_" #name "():args");                    \
1221                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1222                     "atomic_load_" #name "():ptr");                     \
1223                 kmsan_init_ret(sizeof(type));                           \
1224                 return (atomic_load_##name(ptr));                       \
1225         }
1226
1227 #define MSAN_ATOMIC_FUNC_LOAD(name, type)                               \
1228         _MSAN_ATOMIC_FUNC_LOAD(name, type)                              \
1229         _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type)
1230
1231 #define _MSAN_ATOMIC_FUNC_STORE(name, type)                             \
1232         void kmsan_atomic_store_##name(volatile type *ptr, type val)    \
1233         {                                                               \
1234                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1235                     "atomic_store_" #name "():args");                   \
1236                 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED,   \
1237                     sizeof(type));                                      \
1238                 atomic_store_##name(ptr, val);                          \
1239         }
1240
1241 #define MSAN_ATOMIC_FUNC_STORE(name, type)                              \
1242         _MSAN_ATOMIC_FUNC_STORE(name, type)                             \
1243         _MSAN_ATOMIC_FUNC_STORE(rel_##name, type)
1244
1245 MSAN_ATOMIC_FUNC_ADD(8, uint8_t);
1246 MSAN_ATOMIC_FUNC_ADD(16, uint16_t);
1247 MSAN_ATOMIC_FUNC_ADD(32, uint32_t);
1248 MSAN_ATOMIC_FUNC_ADD(64, uint64_t);
1249 MSAN_ATOMIC_FUNC_ADD(int, u_int);
1250 MSAN_ATOMIC_FUNC_ADD(long, u_long);
1251 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
1252
1253 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
1254 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
1255 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
1256 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
1257 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
1258 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
1259 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
1260
1261 MSAN_ATOMIC_FUNC_SET(8, uint8_t);
1262 MSAN_ATOMIC_FUNC_SET(16, uint16_t);
1263 MSAN_ATOMIC_FUNC_SET(32, uint32_t);
1264 MSAN_ATOMIC_FUNC_SET(64, uint64_t);
1265 MSAN_ATOMIC_FUNC_SET(int, u_int);
1266 MSAN_ATOMIC_FUNC_SET(long, u_long);
1267 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
1268
1269 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
1270 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
1271 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
1272 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
1273 MSAN_ATOMIC_FUNC_CLEAR(int, u_int);
1274 MSAN_ATOMIC_FUNC_CLEAR(long, u_long);
1275 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
1276
1277 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
1278 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
1279 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int);
1280 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long);
1281
1282 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
1283 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
1284 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
1285 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
1286 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
1287
1288 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
1289 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
1290 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
1291 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
1292
1293 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
1294 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
1295 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
1296 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
1297
1298 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t);
1299 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t);
1300 MSAN_ATOMIC_FUNC_SWAP(int, u_int);
1301 MSAN_ATOMIC_FUNC_SWAP(long, u_long);
1302 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
1303
1304 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
1305 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
1306 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
1307 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
1308 MSAN_ATOMIC_FUNC_CMPSET(int, u_int);
1309 MSAN_ATOMIC_FUNC_CMPSET(long, u_long);
1310 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
1311
1312 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
1313 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
1314 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
1315 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
1316 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int);
1317 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long);
1318 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
1319
1320 _MSAN_ATOMIC_FUNC_LOAD(bool, bool);
1321 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t);
1322 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t);
1323 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t);
1324 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t);
1325 MSAN_ATOMIC_FUNC_LOAD(char, u_char);
1326 MSAN_ATOMIC_FUNC_LOAD(short, u_short);
1327 MSAN_ATOMIC_FUNC_LOAD(int, u_int);
1328 MSAN_ATOMIC_FUNC_LOAD(long, u_long);
1329 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
1330
1331 _MSAN_ATOMIC_FUNC_STORE(bool, bool);
1332 MSAN_ATOMIC_FUNC_STORE(8, uint8_t);
1333 MSAN_ATOMIC_FUNC_STORE(16, uint16_t);
1334 MSAN_ATOMIC_FUNC_STORE(32, uint32_t);
1335 MSAN_ATOMIC_FUNC_STORE(64, uint64_t);
1336 MSAN_ATOMIC_FUNC_STORE(char, u_char);
1337 MSAN_ATOMIC_FUNC_STORE(short, u_short);
1338 MSAN_ATOMIC_FUNC_STORE(int, u_int);
1339 MSAN_ATOMIC_FUNC_STORE(long, u_long);
1340 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
1341
1342 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq);
1343 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel);
1344 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
1345 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
1346
1347 void
1348 kmsan_atomic_interrupt_fence(void)
1349 {
1350         atomic_interrupt_fence();
1351 }
1352
1353 /* -------------------------------------------------------------------------- */
1354
1355 #include <sys/bus.h>
1356 #include <machine/bus.h>
1357 #include <sys/bus_san.h>
1358
1359 int
1360 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
1361     int flags, bus_space_handle_t *handlep)
1362 {
1363         return (bus_space_map(tag, hnd, size, flags, handlep));
1364 }
1365
1366 void
1367 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
1368     bus_size_t size)
1369 {
1370         bus_space_unmap(tag, hnd, size);
1371 }
1372
1373 int
1374 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
1375     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
1376 {
1377         return (bus_space_subregion(tag, hnd, offset, size, handlep));
1378 }
1379
1380 void
1381 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
1382     bus_size_t size)
1383 {
1384         bus_space_free(tag, hnd, size);
1385 }
1386
1387 void
1388 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
1389     bus_size_t offset, bus_size_t size, int flags)
1390 {
1391         bus_space_barrier(tag, hnd, offset, size, flags);
1392 }
1393
1394 /* XXXMJ x86-specific */
1395 #define MSAN_BUS_READ_FUNC(func, width, type)                           \
1396         type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
1397             bus_space_handle_t hnd, bus_size_t offset)                  \
1398         {                                                               \
1399                 type ret;                                               \
1400                 if ((tag) != X86_BUS_SPACE_IO)                          \
1401                         kmsan_shadow_fill((uintptr_t)(hnd + offset),    \
1402                             KMSAN_STATE_INITED, (width));               \
1403                 ret = bus_space_read##func##_##width(tag, hnd, offset); \
1404                 kmsan_init_ret(sizeof(type));                           \
1405                 return (ret);                                           \
1406         }                                                               \
1407
1408 #define MSAN_BUS_READ_PTR_FUNC(func, width, type)                       \
1409         void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
1410             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
1411             bus_size_t count)                                           \
1412         {                                                               \
1413                 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED,   \
1414                     (width) * count);                                   \
1415                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
1416                     count);                                             \
1417         }
1418
1419 MSAN_BUS_READ_FUNC(, 1, uint8_t)
1420 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
1421 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
1422 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
1423 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
1424 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
1425
1426 MSAN_BUS_READ_FUNC(, 2, uint16_t)
1427 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
1428 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
1429 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
1430 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
1431 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
1432
1433 MSAN_BUS_READ_FUNC(, 4, uint32_t)
1434 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
1435 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
1436 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
1437 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
1438 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
1439
1440 MSAN_BUS_READ_FUNC(, 8, uint64_t)
1441
1442 #define MSAN_BUS_WRITE_FUNC(func, width, type)                          \
1443         void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
1444             bus_space_handle_t hnd, bus_size_t offset, type value)      \
1445         {                                                               \
1446                 bus_space_write##func##_##width(tag, hnd, offset, value);\
1447         }                                                               \
1448
1449 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
1450         void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
1451             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
1452             bus_size_t count)                                           \
1453         {                                                               \
1454                 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
1455                     "bus_space_write()");                               \
1456                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
1457                     count);                                             \
1458         }
1459
1460 MSAN_BUS_WRITE_FUNC(, 1, uint8_t)
1461 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
1462 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
1463 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
1464 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
1465 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
1466
1467 MSAN_BUS_WRITE_FUNC(, 2, uint16_t)
1468 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
1469 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
1470 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
1471 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1472 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1473
1474 MSAN_BUS_WRITE_FUNC(, 4, uint32_t)
1475 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1476 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1477 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1478 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1479 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1480
1481 MSAN_BUS_WRITE_FUNC(, 8, uint64_t)
1482
1483 #define MSAN_BUS_SET_FUNC(func, width, type)                            \
1484         void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
1485             bus_space_handle_t hnd, bus_size_t offset, type value,      \
1486             bus_size_t count)                                           \
1487         {                                                               \
1488                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1489                     count);                                             \
1490         }
1491
1492 MSAN_BUS_SET_FUNC(multi, 1, uint8_t)
1493 MSAN_BUS_SET_FUNC(region, 1, uint8_t)
1494 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1495 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1496
1497 MSAN_BUS_SET_FUNC(multi, 2, uint16_t)
1498 MSAN_BUS_SET_FUNC(region, 2, uint16_t)
1499 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1500 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1501
1502 MSAN_BUS_SET_FUNC(multi, 4, uint32_t)
1503 MSAN_BUS_SET_FUNC(region, 4, uint32_t)
1504 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1505 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1506
1507 /* -------------------------------------------------------------------------- */
1508
1509 void
1510 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op)
1511 {
1512         /*
1513          * Some drivers, e.g., nvme, use the same code path for loading device
1514          * read and write requests, and will thus specify both flags.  In this
1515          * case we should not do any checking since it will generally lead to
1516          * false positives.
1517          */
1518         if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) ==
1519             BUS_DMASYNC_PREWRITE) {
1520                 switch (desc->md_type) {
1521                 case MEMDESC_VADDR:
1522                         kmsan_check(desc->u.md_vaddr, desc->md_len,
1523                             "dmasync");
1524                         break;
1525                 case MEMDESC_MBUF:
1526                         kmsan_check_mbuf(desc->u.md_mbuf, "dmasync");
1527                         break;
1528                 case 0:
1529                         break;
1530                 default:
1531                         kmsan_panic("%s: unhandled memdesc type %d", __func__,
1532                             desc->md_type);
1533                 }
1534         }
1535         if ((op & BUS_DMASYNC_POSTREAD) != 0) {
1536                 switch (desc->md_type) {
1537                 case MEMDESC_VADDR:
1538                         kmsan_mark(desc->u.md_vaddr, desc->md_len,
1539                             KMSAN_STATE_INITED);
1540                         break;
1541                 case MEMDESC_MBUF:
1542                         kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED);
1543                         break;
1544                 case 0:
1545                         break;
1546                 default:
1547                         kmsan_panic("%s: unhandled memdesc type %d", __func__,
1548                             desc->md_type);
1549                 }
1550         }
1551 }