sys/kern/subr_msan.c

   1 /*      $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $      */
   2
   3 /*
   4  * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
   5  * All rights reserved.
   6  * Copyright (c) 2021 The FreeBSD Foundation
   7  *
   8  * Portions of this software were developed by Mark Johnston under sponsorship
   9  * from the FreeBSD Foundation.
  10  *
  11  * This code is part of the KMSAN subsystem of the NetBSD kernel.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  23  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  24  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  25  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 #define SAN_RUNTIME
  36
  37 #include <sys/cdefs.h>
  38 #if 0
  39 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $");
  40 #endif
  41
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/bio.h>
  45 #include <sys/buf.h>
  46 #include <sys/conf.h>
  47 #include <sys/kdb.h>
  48 #include <sys/kernel.h>
  49 #include <sys/linker.h>
  50 #include <sys/malloc.h>
  51 #include <sys/mbuf.h>
  52 #include <sys/memdesc.h>
  53 #include <sys/msan.h>
  54 #include <sys/proc.h>
  55 #include <sys/stack.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/uio.h>
  58
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61
  62 #include <machine/msan.h>
  63 #include <machine/stdarg.h>
  64
  65 void kmsan_init_arg(size_t);
  66 void kmsan_init_ret(size_t);
  67
  68 /* -------------------------------------------------------------------------- */
  69
  70 /*
  71  * Part of the compiler ABI.
  72  */
  73
  74 typedef struct {
  75         uint8_t *shad;
  76         msan_orig_t *orig;
  77 } msan_meta_t;
  78
  79 #define MSAN_PARAM_SIZE         800
  80 #define MSAN_RETVAL_SIZE        800
  81 typedef struct {
  82         uint8_t param_shadow[MSAN_PARAM_SIZE];
  83         uint8_t retval_shadow[MSAN_RETVAL_SIZE];
  84         uint8_t va_arg_shadow[MSAN_PARAM_SIZE];
  85         uint8_t va_arg_origin[MSAN_PARAM_SIZE];
  86         uint64_t va_arg_overflow_size;
  87         msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)];
  88         msan_orig_t retval_origin;
  89 } msan_tls_t;
  90
  91 /* -------------------------------------------------------------------------- */
  92
  93 #define MSAN_NCONTEXT   4
  94 #define MSAN_ORIG_MASK  (~0x3)
  95
  96 typedef struct kmsan_td {
  97         size_t ctx;
  98         msan_tls_t tls[MSAN_NCONTEXT];
  99 } msan_td_t;
 100
 101 static msan_tls_t dummy_tls;
 102
 103 /*
 104  * Use separate dummy regions for loads and stores: stores may mark the region
 105  * as uninitialized, and that can trigger false positives.
 106  */
 107 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
 108 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
 109 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
 110 static msan_td_t msan_thread0;
 111 static bool kmsan_enabled __read_mostly;
 112
 113 static bool kmsan_reporting = false;
 114
 115 /*
 116  * Avoid clobbering any thread-local state before we panic.
 117  */
 118 #define kmsan_panic(f, ...) do {                        \
 119         kmsan_enabled = false;                          \
 120         panic(f, __VA_ARGS__);                          \
 121 } while (0)
 122
 123 #define REPORT(f, ...) do {                             \
 124         if (panic_on_violation) {                       \
 125                 kmsan_panic(f, __VA_ARGS__);            \
 126         } else {                                        \
 127                 struct stack st;                        \
 128                                                         \
 129                 stack_save(&st);                        \
 130                 printf(f "\n", __VA_ARGS__);            \
 131                 stack_print_ddb(&st);                   \
 132         }                                               \
 133 } while (0)
 134
 135 FEATURE(kmsan, "Kernel memory sanitizer");
 136
 137 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 138     "KMSAN options");
 139
 140 static bool panic_on_violation = 1;
 141 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN,
 142     &panic_on_violation, 0,
 143     "Panic if an invalid access is detected");
 144
 145 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer");
 146
 147 /* -------------------------------------------------------------------------- */
 148
 149 static inline const char *
 150 kmsan_orig_name(int type)
 151 {
 152         switch (type) {
 153         case KMSAN_TYPE_STACK:
 154                 return ("stack");
 155         case KMSAN_TYPE_KMEM:
 156                 return ("kmem");
 157         case KMSAN_TYPE_MALLOC:
 158                 return ("malloc");
 159         case KMSAN_TYPE_UMA:
 160                 return ("UMA");
 161         default:
 162                 return ("unknown");
 163         }
 164 }
 165
 166 static void
 167 kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off,
 168     const char *hook)
 169 {
 170         const char *typename;
 171         char *var, *fn;
 172         uintptr_t ptr;
 173         long foff;
 174         char buf[128];
 175         int type;
 176
 177         if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
 178                 return;
 179
 180         kmsan_reporting = true;
 181         __compiler_membar();
 182
 183         if (*orig == 0) {
 184                 REPORT("MSan: Uninitialized memory in %s, offset %zu",
 185                     hook, off);
 186                 goto out;
 187         }
 188
 189         kmsan_md_orig_decode(*orig, &type, &ptr);
 190         typename = kmsan_orig_name(type);
 191
 192         if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
 193             sizeof(buf), &foff) == 0) {
 194                 REPORT("MSan: Uninitialized %s memory in %s, "
 195                     "offset %zu/%zu, addr %p, from %s+%#lx",
 196                     typename, hook, off, size, addr, buf, foff);
 197         } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
 198                 /*
 199                  * The format of the string is: "----var@function". Parse it to
 200                  * display a nice warning.
 201                  */
 202                 var = (char *)ptr + 4;
 203                 strlcpy(buf, var, sizeof(buf));
 204                 var = buf;
 205                 fn = strchr(buf, '@');
 206                 *fn++ = '\0';
 207                 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, "
 208                     "variable '%s' from %s", typename, hook, off, var, fn);
 209         } else {
 210                 REPORT("MSan: Uninitialized %s memory in %s, "
 211                     "offset %zu/%zu, addr %p, PC %p",
 212                     typename, hook, off, size, addr, (void *)ptr);
 213         }
 214
 215 out:
 216         __compiler_membar();
 217         kmsan_reporting = false;
 218 }
 219
 220 static void
 221 kmsan_report_inline(msan_orig_t orig, unsigned long pc)
 222 {
 223         const char *typename;
 224         char *var, *fn;
 225         uintptr_t ptr;
 226         char buf[128];
 227         long foff;
 228         int type;
 229
 230         if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
 231                 return;
 232
 233         kmsan_reporting = true;
 234         __compiler_membar();
 235
 236         if (orig == 0) {
 237                 REPORT("MSan: uninitialized variable in %p", (void *)pc);
 238                 goto out;
 239         }
 240
 241         kmsan_md_orig_decode(orig, &type, &ptr);
 242         typename = kmsan_orig_name(type);
 243
 244         if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
 245             sizeof(buf), &foff) == 0) {
 246                 REPORT("MSan: Uninitialized %s memory from %s+%#lx",
 247                     typename, buf, foff);
 248         } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
 249                 /*
 250                  * The format of the string is: "----var@function". Parse it to
 251                  * display a nice warning.
 252                  */
 253                 var = (char *)ptr + 4;
 254                 strlcpy(buf, var, sizeof(buf));
 255                 var = buf;
 256                 fn = strchr(buf, '@');
 257                 *fn++ = '\0';
 258                 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn);
 259         } else {
 260                 REPORT("MSan: Uninitialized %s memory, origin %x",
 261                     typename, orig);
 262         }
 263
 264 out:
 265         __compiler_membar();
 266         kmsan_reporting = false;
 267 }
 268
 269 /* -------------------------------------------------------------------------- */
 270
 271 static inline msan_meta_t
 272 kmsan_meta_get(const void *addr, size_t size, const bool write)
 273 {
 274         msan_meta_t ret;
 275
 276         if (__predict_false(!kmsan_enabled)) {
 277                 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
 278                 ret.orig = (msan_orig_t *)msan_dummy_orig;
 279         } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) {
 280                 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
 281                 ret.orig = (msan_orig_t *)msan_dummy_orig;
 282         } else {
 283                 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr);
 284                 ret.orig =
 285                     (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
 286                 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig &
 287                     MSAN_ORIG_MASK);
 288         }
 289
 290         return (ret);
 291 }
 292
 293 static inline void
 294 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size)
 295 {
 296         msan_orig_t *orig;
 297         size_t i;
 298
 299         if (__predict_false(!kmsan_enabled))
 300                 return;
 301         if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr)))
 302                 return;
 303
 304         orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
 305         size += ((uintptr_t)orig & (sizeof(*orig) - 1));
 306         orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
 307
 308         for (i = 0; i < size; i += 4) {
 309                 orig[i / 4] = o;
 310         }
 311 }
 312
 313 static inline void
 314 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size)
 315 {
 316         uint8_t *shad;
 317
 318         if (__predict_false(!kmsan_enabled))
 319                 return;
 320         if (__predict_false(kmsan_md_unsupported(addr)))
 321                 return;
 322
 323         shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
 324         __builtin_memset(shad, c, size);
 325 }
 326
 327 static inline void
 328 kmsan_meta_copy(void *dst, const void *src, size_t size)
 329 {
 330         uint8_t *orig_src, *orig_dst;
 331         uint8_t *shad_src, *shad_dst;
 332         msan_orig_t *_src, *_dst;
 333         size_t i;
 334
 335         if (__predict_false(!kmsan_enabled))
 336                 return;
 337         if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst)))
 338                 return;
 339         if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) {
 340                 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size);
 341                 return;
 342         }
 343
 344         shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src);
 345         shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst);
 346         __builtin_memmove(shad_dst, shad_src, size);
 347
 348         orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src);
 349         orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst);
 350         for (i = 0; i < size; i++) {
 351                 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK);
 352                 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK);
 353                 *_dst = *_src;
 354                 orig_src++;
 355                 orig_dst++;
 356         }
 357 }
 358
 359 static inline void
 360 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook)
 361 {
 362         msan_orig_t *orig;
 363         uint8_t *shad;
 364         size_t i;
 365
 366         if (__predict_false(!kmsan_enabled))
 367                 return;
 368         if (__predict_false(kmsan_md_unsupported(addr)))
 369                 return;
 370
 371         shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
 372         for (i = 0; i < size; i++) {
 373                 if (__predict_true(shad[i] == 0))
 374                         continue;
 375                 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)&shad[i]);
 376                 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
 377                 kmsan_report_hook((const char *)addr + i, orig, size, i, hook);
 378                 break;
 379         }
 380 }
 381
 382 void
 383 kmsan_init_arg(size_t n)
 384 {
 385         msan_td_t *mtd;
 386         uint8_t *arg;
 387
 388         if (__predict_false(!kmsan_enabled))
 389                 return;
 390         if (__predict_false(curthread == NULL))
 391                 return;
 392         mtd = curthread->td_kmsan;
 393         arg = mtd->tls[mtd->ctx].param_shadow;
 394         __builtin_memset(arg, 0, n);
 395 }
 396
 397 void
 398 kmsan_init_ret(size_t n)
 399 {
 400         msan_td_t *mtd;
 401         uint8_t *arg;
 402
 403         if (__predict_false(!kmsan_enabled))
 404                 return;
 405         if (__predict_false(curthread == NULL))
 406                 return;
 407         mtd = curthread->td_kmsan;
 408         arg = mtd->tls[mtd->ctx].retval_shadow;
 409         __builtin_memset(arg, 0, n);
 410 }
 411
 412 static void
 413 kmsan_check_arg(size_t size, const char *hook)
 414 {
 415         msan_orig_t *orig;
 416         msan_td_t *mtd;
 417         uint8_t *arg;
 418         size_t ctx, i;
 419
 420         if (__predict_false(!kmsan_enabled))
 421                 return;
 422         if (__predict_false(curthread == NULL))
 423                 return;
 424         mtd = curthread->td_kmsan;
 425         ctx = mtd->ctx;
 426         arg = mtd->tls[ctx].param_shadow;
 427
 428         for (i = 0; i < size; i++) {
 429                 if (__predict_true(arg[i] == 0))
 430                         continue;
 431                 orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)];
 432                 kmsan_report_hook((const char *)arg + i, orig, size, i, hook);
 433                 break;
 434         }
 435 }
 436
 437 void
 438 kmsan_thread_alloc(struct thread *td)
 439 {
 440         msan_td_t *mtd;
 441
 442         if (!kmsan_enabled)
 443                 return;
 444
 445         mtd = td->td_kmsan;
 446         if (mtd == NULL) {
 447                 /* We might be recycling a thread. */
 448                 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) +
 449                     sizeof(int));
 450                 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK);
 451         }
 452         kmsan_memset(mtd, 0, sizeof(*mtd));
 453         mtd->ctx = 0;
 454
 455         if (td->td_kstack != 0)
 456                 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages),
 457                     KMSAN_STATE_UNINIT);
 458
 459         td->td_kmsan = mtd;
 460 }
 461
 462 void
 463 kmsan_thread_free(struct thread *td)
 464 {
 465         msan_td_t *mtd;
 466
 467         if (!kmsan_enabled)
 468                 return;
 469         if (__predict_false(td == curthread))
 470                 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__);
 471
 472         mtd = td->td_kmsan;
 473         kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *));
 474         free(mtd, M_KMSAN);
 475         td->td_kmsan = NULL;
 476 }
 477
 478 void kmsan_intr_enter(void);
 479 void kmsan_intr_leave(void);
 480
 481 void
 482 kmsan_intr_enter(void)
 483 {
 484         msan_td_t *mtd;
 485
 486         if (__predict_false(!kmsan_enabled))
 487                 return;
 488
 489         mtd = curthread->td_kmsan;
 490         mtd->ctx++;
 491         if (__predict_false(mtd->ctx >= MSAN_NCONTEXT))
 492                 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
 493 }
 494
 495 void
 496 kmsan_intr_leave(void)
 497 {
 498         msan_td_t *mtd;
 499
 500         if (__predict_false(!kmsan_enabled))
 501                 return;
 502
 503         mtd = curthread->td_kmsan;
 504         if (__predict_false(mtd->ctx == 0))
 505                 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
 506         mtd->ctx--;
 507 }
 508
 509 /* -------------------------------------------------------------------------- */
 510
 511 void
 512 kmsan_shadow_map(vm_offset_t addr, size_t size)
 513 {
 514         size_t npages, i;
 515         vm_offset_t va;
 516
 517         MPASS(addr % PAGE_SIZE == 0);
 518         MPASS(size % PAGE_SIZE == 0);
 519
 520         if (!kmsan_enabled)
 521                 return;
 522
 523         npages = atop(size);
 524
 525         va = kmsan_md_addr_to_shad(addr);
 526         for (i = 0; i < npages; i++) {
 527                 pmap_san_enter(va + ptoa(i));
 528         }
 529
 530         va = kmsan_md_addr_to_orig(addr);
 531         for (i = 0; i < npages; i++) {
 532                 pmap_san_enter(va + ptoa(i));
 533         }
 534 }
 535
 536 void
 537 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc)
 538 {
 539         msan_orig_t orig;
 540
 541         orig = kmsan_md_orig_encode(type, pc);
 542         kmsan_origin_fill(addr, orig, size);
 543 }
 544
 545 void
 546 kmsan_mark(const void *addr, size_t size, uint8_t c)
 547 {
 548         kmsan_shadow_fill((uintptr_t)addr, c, size);
 549 }
 550
 551 void
 552 kmsan_mark_bio(const struct bio *bp, uint8_t c)
 553 {
 554         kmsan_mark(bp->bio_data, bp->bio_length, c);
 555 }
 556
 557 void
 558 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c)
 559 {
 560         do {
 561                 if ((m->m_flags & M_EXTPG) == 0)
 562                         kmsan_mark(m->m_data, m->m_len, c);
 563                 m = m->m_next;
 564         } while (m != NULL);
 565 }
 566
 567 void
 568 kmsan_check(const void *p, size_t sz, const char *descr)
 569 {
 570         kmsan_shadow_check((uintptr_t)p, sz, descr);
 571 }
 572
 573 void
 574 kmsan_check_bio(const struct bio *bp, const char *descr)
 575 {
 576         kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr);
 577 }
 578
 579 void
 580 kmsan_check_mbuf(const struct mbuf *m, const char *descr)
 581 {
 582         do {
 583                 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr);
 584         } while ((m = m->m_next) != NULL);
 585 }
 586
 587 void
 588 kmsan_init(void)
 589 {
 590         int disabled;
 591
 592         disabled = 0;
 593         TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled);
 594         if (disabled)
 595                 return;
 596
 597         /* Initialize the TLS for curthread. */
 598         msan_thread0.ctx = 0;
 599         thread0.td_kmsan = &msan_thread0;
 600
 601         /* Now officially enabled. */
 602         kmsan_enabled = true;
 603 }
 604
 605 /* -------------------------------------------------------------------------- */
 606
 607 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
 608 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
 609
 610 msan_meta_t
 611 __msan_metadata_ptr_for_load_n(void *addr, size_t size)
 612 {
 613         return (kmsan_meta_get(addr, size, false));
 614 }
 615
 616 msan_meta_t
 617 __msan_metadata_ptr_for_store_n(void *addr, size_t size)
 618 {
 619         return (kmsan_meta_get(addr, size, true));
 620 }
 621
 622 #define MSAN_META_FUNC(size)                                            \
 623         msan_meta_t __msan_metadata_ptr_for_load_##size(void *);        \
 624         msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr)     \
 625         {                                                               \
 626                 return (kmsan_meta_get(addr, size, false));             \
 627         }                                                               \
 628         msan_meta_t __msan_metadata_ptr_for_store_##size(void *);       \
 629         msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr)    \
 630         {                                                               \
 631                 return (kmsan_meta_get(addr, size, true));              \
 632         }
 633
 634 MSAN_META_FUNC(1)
 635 MSAN_META_FUNC(2)
 636 MSAN_META_FUNC(4)
 637 MSAN_META_FUNC(8)
 638
 639 void __msan_instrument_asm_store(const void *, size_t);
 640 msan_orig_t __msan_chain_origin(msan_orig_t);
 641 void __msan_poison(const void *, size_t);
 642 void __msan_unpoison(const void *, size_t);
 643 void __msan_poison_alloca(const void *, uint64_t, const char *);
 644 void __msan_unpoison_alloca(const void *, uint64_t);
 645 void __msan_warning(msan_orig_t);
 646 msan_tls_t *__msan_get_context_state(void);
 647
 648 void
 649 __msan_instrument_asm_store(const void *addr, size_t size)
 650 {
 651         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
 652 }
 653
 654 msan_orig_t
 655 __msan_chain_origin(msan_orig_t origin)
 656 {
 657         return (origin);
 658 }
 659
 660 void
 661 __msan_poison(const void *addr, size_t size)
 662 {
 663         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
 664 }
 665
 666 void
 667 __msan_unpoison(const void *addr, size_t size)
 668 {
 669         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
 670 }
 671
 672 void
 673 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr)
 674 {
 675         msan_orig_t orig;
 676
 677         orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
 678         kmsan_origin_fill(addr, orig, size);
 679         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
 680 }
 681
 682 void
 683 __msan_unpoison_alloca(const void *addr, uint64_t size)
 684 {
 685         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
 686 }
 687
 688 void
 689 __msan_warning(msan_orig_t origin)
 690 {
 691         if (__predict_false(!kmsan_enabled))
 692                 return;
 693         kmsan_report_inline(origin, KMSAN_RET_ADDR);
 694 }
 695
 696 msan_tls_t *
 697 __msan_get_context_state(void)
 698 {
 699         msan_td_t *mtd;
 700
 701         /*
 702          * When APs are started, they execute some C code before curthread is
 703          * set.  We have to handle that here.
 704          */
 705         if (__predict_false(!kmsan_enabled || curthread == NULL))
 706                 return (&dummy_tls);
 707         mtd = curthread->td_kmsan;
 708         return (&mtd->tls[mtd->ctx]);
 709 }
 710
 711 /* -------------------------------------------------------------------------- */
 712
 713 /*
 714  * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
 715  * initialized areas properly.
 716  */
 717
 718 void *
 719 kmsan_memcpy(void *dst, const void *src, size_t len)
 720 {
 721         /* No kmsan_check_arg, because inlined. */
 722         kmsan_init_ret(sizeof(void *));
 723         if (__predict_true(len != 0)) {
 724                 kmsan_meta_copy(dst, src, len);
 725         }
 726         return (__builtin_memcpy(dst, src, len));
 727 }
 728
 729 int
 730 kmsan_memcmp(const void *b1, const void *b2, size_t len)
 731 {
 732         const uint8_t *_b1 = b1, *_b2 = b2;
 733         size_t i;
 734
 735         kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len),
 736             "memcmp():args");
 737         kmsan_init_ret(sizeof(int));
 738
 739         for (i = 0; i < len; i++) {
 740                 if (*_b1 != *_b2) {
 741                         kmsan_shadow_check((uintptr_t)b1, i + 1,
 742                             "memcmp():arg1");
 743                         kmsan_shadow_check((uintptr_t)b2, i + 1,
 744                             "memcmp():arg2");
 745                         return (*_b1 - *_b2);
 746                 }
 747                 _b1++, _b2++;
 748         }
 749
 750         return (0);
 751 }
 752
 753 void *
 754 kmsan_memset(void *dst, int c, size_t len)
 755 {
 756         /* No kmsan_check_arg, because inlined. */
 757         kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len);
 758         kmsan_init_ret(sizeof(void *));
 759         return (__builtin_memset(dst, c, len));
 760 }
 761
 762 void *
 763 kmsan_memmove(void *dst, const void *src, size_t len)
 764 {
 765         /* No kmsan_check_arg, because inlined. */
 766         if (__predict_true(len != 0)) {
 767                 kmsan_meta_copy(dst, src, len);
 768         }
 769         kmsan_init_ret(sizeof(void *));
 770         return (__builtin_memmove(dst, src, len));
 771 }
 772
 773 __strong_reference(kmsan_memcpy, __msan_memcpy);
 774 __strong_reference(kmsan_memset, __msan_memset);
 775 __strong_reference(kmsan_memmove, __msan_memmove);
 776
 777 char *
 778 kmsan_strcpy(char *dst, const char *src)
 779 {
 780         const char *_src = src;
 781         char *_dst = dst;
 782         size_t len = 0;
 783
 784         kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args");
 785
 786         while (1) {
 787                 len++;
 788                 *dst = *src;
 789                 if (*src == '\0')
 790                         break;
 791                 src++, dst++;
 792         }
 793
 794         kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2");
 795         kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len);
 796         kmsan_init_ret(sizeof(char *));
 797         return (_dst);
 798 }
 799
 800 int
 801 kmsan_strcmp(const char *s1, const char *s2)
 802 {
 803         const char *_s1 = s1, *_s2 = s2;
 804         size_t len = 0;
 805
 806         kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args");
 807         kmsan_init_ret(sizeof(int));
 808
 809         while (1) {
 810                 len++;
 811                 if (*s1 != *s2)
 812                         break;
 813                 if (*s1 == '\0') {
 814                         kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
 815                         kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
 816                         return (0);
 817                 }
 818                 s1++, s2++;
 819         }
 820
 821         kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
 822         kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
 823
 824         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 825 }
 826
 827 size_t
 828 kmsan_strlen(const char *str)
 829 {
 830         const char *s;
 831
 832         kmsan_check_arg(sizeof(str), "strlen():args");
 833
 834         s = str;
 835         while (1) {
 836                 if (*s == '\0')
 837                         break;
 838                 s++;
 839         }
 840
 841         kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1");
 842         kmsan_init_ret(sizeof(size_t));
 843         return (s - str);
 844 }
 845
 846 int     kmsan_copyin(const void *, void *, size_t);
 847 int     kmsan_copyout(const void *, void *, size_t);
 848 int     kmsan_copyinstr(const void *, void *, size_t, size_t *);
 849
 850 int
 851 kmsan_copyin(const void *uaddr, void *kaddr, size_t len)
 852 {
 853         int ret;
 854
 855         kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len),
 856             "copyin():args");
 857         ret = copyin(uaddr, kaddr, len);
 858         if (ret == 0)
 859                 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len);
 860         kmsan_init_ret(sizeof(int));
 861         return (ret);
 862 }
 863
 864 int
 865 kmsan_copyout(const void *kaddr, void *uaddr, size_t len)
 866 {
 867         kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len),
 868             "copyout():args");
 869         kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1");
 870         kmsan_init_ret(sizeof(int));
 871         return (copyout(kaddr, uaddr, len));
 872 }
 873
 874 int
 875 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 876 {
 877         size_t _done;
 878         int ret;
 879
 880         kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) +
 881             sizeof(len) + sizeof(done), "copyinstr():args");
 882         ret = copyinstr(uaddr, kaddr, len, &_done);
 883         if (ret == 0)
 884                 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done);
 885         if (done != NULL) {
 886                 *done = _done;
 887                 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t));
 888         }
 889         kmsan_init_ret(sizeof(int));
 890         return (ret);
 891 }
 892
 893 /* -------------------------------------------------------------------------- */
 894
 895 int
 896 kmsan_fubyte(volatile const void *base)
 897 {
 898         int ret;
 899
 900         kmsan_check_arg(sizeof(base), "fubyte(): args");
 901         ret = fubyte(base);
 902         kmsan_init_ret(sizeof(int));
 903         return (ret);
 904 }
 905
 906 int
 907 kmsan_fuword16(volatile const void *base)
 908 {
 909         int ret;
 910
 911         kmsan_check_arg(sizeof(base), "fuword16(): args");
 912         ret = fuword16(base);
 913         kmsan_init_ret(sizeof(int));
 914         return (ret);
 915 }
 916
 917 int
 918 kmsan_fueword(volatile const void *base, long *val)
 919 {
 920         int ret;
 921
 922         kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args");
 923         ret = fueword(base, val);
 924         if (ret == 0)
 925                 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
 926                     sizeof(*val));
 927         kmsan_init_ret(sizeof(int));
 928         return (ret);
 929 }
 930
 931 int
 932 kmsan_fueword32(volatile const void *base, int32_t *val)
 933 {
 934         int ret;
 935
 936         kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args");
 937         ret = fueword32(base, val);
 938         if (ret == 0)
 939                 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
 940                     sizeof(*val));
 941         kmsan_init_ret(sizeof(int));
 942         return (ret);
 943 }
 944
 945 int
 946 kmsan_fueword64(volatile const void *base, int64_t *val)
 947 {
 948         int ret;
 949
 950         kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args");
 951         ret = fueword64(base, val);
 952         if (ret == 0)
 953                 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
 954                     sizeof(*val));
 955         kmsan_init_ret(sizeof(int));
 956         return (ret);
 957 }
 958
 959 int
 960 kmsan_subyte(volatile void *base, int byte)
 961 {
 962         int ret;
 963
 964         kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args");
 965         ret = subyte(base, byte);
 966         kmsan_init_ret(sizeof(int));
 967         return (ret);
 968 }
 969
 970 int
 971 kmsan_suword(volatile void *base, long word)
 972 {
 973         int ret;
 974
 975         kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args");
 976         ret = suword(base, word);
 977         kmsan_init_ret(sizeof(int));
 978         return (ret);
 979 }
 980
 981 int
 982 kmsan_suword16(volatile void *base, int word)
 983 {
 984         int ret;
 985
 986         kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args");
 987         ret = suword16(base, word);
 988         kmsan_init_ret(sizeof(int));
 989         return (ret);
 990 }
 991
 992 int
 993 kmsan_suword32(volatile void *base, int32_t word)
 994 {
 995         int ret;
 996
 997         kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args");
 998         ret = suword32(base, word);
 999         kmsan_init_ret(sizeof(int));
1000         return (ret);
1001 }
1002
1003 int
1004 kmsan_suword64(volatile void *base, int64_t word)
1005 {
1006         int ret;
1007
1008         kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args");
1009         ret = suword64(base, word);
1010         kmsan_init_ret(sizeof(int));
1011         return (ret);
1012 }
1013
1014 int
1015 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
1016     uint32_t newval)
1017 {
1018         int ret;
1019
1020         kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1021             sizeof(newval), "casueword32(): args");
1022         ret = casueword32(base, oldval, oldvalp, newval);
1023         kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1024             sizeof(*oldvalp));
1025         kmsan_init_ret(sizeof(int));
1026         return (ret);
1027 }
1028
1029 int
1030 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
1031     u_long newval)
1032 {
1033         int ret;
1034
1035         kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1036             sizeof(newval), "casueword32(): args");
1037         ret = casueword(base, oldval, oldvalp, newval);
1038         kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1039             sizeof(*oldvalp));
1040         kmsan_init_ret(sizeof(int));
1041         return (ret);
1042 }
1043
1044 /* -------------------------------------------------------------------------- */
1045
1046 #include <machine/atomic.h>
1047 #include <sys/atomic_san.h>
1048
1049 #define _MSAN_ATOMIC_FUNC_ADD(name, type)                               \
1050         void kmsan_atomic_add_##name(volatile type *ptr, type val)      \
1051         {                                                               \
1052                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1053                     "atomic_add_" #name "():args");                     \
1054                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1055                     "atomic_add_" #name "():ptr");                      \
1056                 atomic_add_##name(ptr, val);                            \
1057         }
1058
1059 #define MSAN_ATOMIC_FUNC_ADD(name, type)                                \
1060         _MSAN_ATOMIC_FUNC_ADD(name, type)                               \
1061         _MSAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
1062         _MSAN_ATOMIC_FUNC_ADD(rel_##name, type)
1063
1064 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
1065         void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \
1066         {                                                               \
1067                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1068                     "atomic_subtract_" #name "():args");                \
1069                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1070                     "atomic_subtract_" #name "():ptr");                 \
1071                 atomic_subtract_##name(ptr, val);                       \
1072         }
1073
1074 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
1075         _MSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
1076         _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
1077         _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
1078
1079 #define _MSAN_ATOMIC_FUNC_SET(name, type)                               \
1080         void kmsan_atomic_set_##name(volatile type *ptr, type val)      \
1081         {                                                               \
1082                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1083                     "atomic_set_" #name "():args");                     \
1084                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1085                     "atomic_set_" #name "():ptr");                      \
1086                 atomic_set_##name(ptr, val);                            \
1087         }
1088
1089 #define MSAN_ATOMIC_FUNC_SET(name, type)                                \
1090         _MSAN_ATOMIC_FUNC_SET(name, type)                               \
1091         _MSAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
1092         _MSAN_ATOMIC_FUNC_SET(rel_##name, type)
1093
1094 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
1095         void kmsan_atomic_clear_##name(volatile type *ptr, type val)    \
1096         {                                                               \
1097                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1098                     "atomic_clear_" #name "():args");                   \
1099                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1100                     "atomic_clear_" #name "():ptr");                    \
1101                 atomic_clear_##name(ptr, val);                          \
1102         }
1103
1104 #define MSAN_ATOMIC_FUNC_CLEAR(name, type)                              \
1105         _MSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
1106         _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
1107         _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
1108
1109 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
1110         type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
1111         {                                                               \
1112                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1113                     "atomic_fetchadd_" #name "():args");                \
1114                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1115                     "atomic_fetchadd_" #name "():ptr");                 \
1116                 kmsan_init_ret(sizeof(type));                           \
1117                 return (atomic_fetchadd_##name(ptr, val));              \
1118         }
1119
1120 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
1121         type kmsan_atomic_readandclear_##name(volatile type *ptr)       \
1122         {                                                               \
1123                 kmsan_check_arg(sizeof(ptr),                            \
1124                     "atomic_readandclear_" #name "():args");            \
1125                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1126                     "atomic_readandclear_" #name "():ptr");             \
1127                 kmsan_init_ret(sizeof(type));                           \
1128                 return (atomic_readandclear_##name(ptr));               \
1129         }
1130
1131 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
1132         int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
1133         {                                                               \
1134                 kmsan_check_arg(sizeof(ptr) + sizeof(v),                \
1135                     "atomic_testandclear_" #name "():args");            \
1136                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1137                     "atomic_testandclear_" #name "():ptr");             \
1138                 kmsan_init_ret(sizeof(int));                            \
1139                 return (atomic_testandclear_##name(ptr, v));            \
1140         }
1141
1142 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
1143         int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \
1144         {                                                               \
1145                 kmsan_check_arg(sizeof(ptr) + sizeof(v),                \
1146                     "atomic_testandset_" #name "():args");              \
1147                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1148                     "atomic_testandset_" #name "():ptr");               \
1149                 kmsan_init_ret(sizeof(int));                            \
1150                 return (atomic_testandset_##name(ptr, v));              \
1151         }
1152
1153 #define MSAN_ATOMIC_FUNC_SWAP(name, type)                               \
1154         type kmsan_atomic_swap_##name(volatile type *ptr, type val)     \
1155         {                                                               \
1156                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1157                     "atomic_swap_" #name "():args");                    \
1158                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1159                     "atomic_swap_" #name "():ptr");                     \
1160                 kmsan_init_ret(sizeof(type));                           \
1161                 return (atomic_swap_##name(ptr, val));                  \
1162         }
1163
1164 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
1165         int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval,   \
1166             type nval)                                                  \
1167         {                                                               \
1168                 kmsan_check_arg(sizeof(ptr) + sizeof(oval) +            \
1169                     sizeof(nval), "atomic_cmpset_" #name "():args");    \
1170                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1171                     "atomic_cmpset_" #name "():ptr");                   \
1172                 kmsan_init_ret(sizeof(int));                            \
1173                 return (atomic_cmpset_##name(ptr, oval, nval));         \
1174         }
1175
1176 #define MSAN_ATOMIC_FUNC_CMPSET(name, type)                             \
1177         _MSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
1178         _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
1179         _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
1180
1181 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
1182         int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
1183             type nval)                                                  \
1184         {                                                               \
1185                 kmsan_check_arg(sizeof(ptr) + sizeof(oval) +            \
1186                     sizeof(nval), "atomic_fcmpset_" #name "():args");   \
1187                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1188                     "atomic_fcmpset_" #name "():ptr");                  \
1189                 kmsan_init_ret(sizeof(int));                            \
1190                 return (atomic_fcmpset_##name(ptr, oval, nval));        \
1191         }
1192
1193 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
1194         _MSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
1195         _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
1196         _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
1197
1198 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
1199         void kmsan_atomic_thread_fence_##name(void)                     \
1200         {                                                               \
1201                 atomic_thread_fence_##name();                           \
1202         }
1203
1204 #define _MSAN_ATOMIC_FUNC_LOAD(name, type)                              \
1205         type kmsan_atomic_load_##name(volatile type *ptr)               \
1206         {                                                               \
1207                 kmsan_check_arg(sizeof(ptr),                            \
1208                     "atomic_load_" #name "():args");                    \
1209                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1210                     "atomic_load_" #name "():ptr");                     \
1211                 kmsan_init_ret(sizeof(type));                           \
1212                 return (atomic_load_##name(ptr));                       \
1213         }
1214
1215 #define MSAN_ATOMIC_FUNC_LOAD(name, type)                               \
1216         _MSAN_ATOMIC_FUNC_LOAD(name, type)                              \
1217         _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type)
1218
1219 #define _MSAN_ATOMIC_FUNC_STORE(name, type)                             \
1220         void kmsan_atomic_store_##name(volatile type *ptr, type val)    \
1221         {                                                               \
1222                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1223                     "atomic_store_" #name "():args");                   \
1224                 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED,   \
1225                     sizeof(type));                                      \
1226                 atomic_store_##name(ptr, val);                          \
1227         }
1228
1229 #define MSAN_ATOMIC_FUNC_STORE(name, type)                              \
1230         _MSAN_ATOMIC_FUNC_STORE(name, type)                             \
1231         _MSAN_ATOMIC_FUNC_STORE(rel_##name, type)
1232
1233 MSAN_ATOMIC_FUNC_ADD(8, uint8_t);
1234 MSAN_ATOMIC_FUNC_ADD(16, uint16_t);
1235 MSAN_ATOMIC_FUNC_ADD(32, uint32_t);
1236 MSAN_ATOMIC_FUNC_ADD(64, uint64_t);
1237 MSAN_ATOMIC_FUNC_ADD(int, u_int);
1238 MSAN_ATOMIC_FUNC_ADD(long, u_long);
1239 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
1240
1241 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
1242 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
1243 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
1244 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
1245 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
1246 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
1247 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
1248
1249 MSAN_ATOMIC_FUNC_SET(8, uint8_t);
1250 MSAN_ATOMIC_FUNC_SET(16, uint16_t);
1251 MSAN_ATOMIC_FUNC_SET(32, uint32_t);
1252 MSAN_ATOMIC_FUNC_SET(64, uint64_t);
1253 MSAN_ATOMIC_FUNC_SET(int, u_int);
1254 MSAN_ATOMIC_FUNC_SET(long, u_long);
1255 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
1256
1257 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
1258 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
1259 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
1260 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
1261 MSAN_ATOMIC_FUNC_CLEAR(int, u_int);
1262 MSAN_ATOMIC_FUNC_CLEAR(long, u_long);
1263 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
1264
1265 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
1266 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
1267 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int);
1268 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long);
1269
1270 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
1271 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
1272 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
1273 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
1274 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
1275
1276 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
1277 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
1278 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
1279 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
1280
1281 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
1282 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
1283 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
1284 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
1285
1286 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t);
1287 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t);
1288 MSAN_ATOMIC_FUNC_SWAP(int, u_int);
1289 MSAN_ATOMIC_FUNC_SWAP(long, u_long);
1290 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
1291
1292 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
1293 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
1294 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
1295 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
1296 MSAN_ATOMIC_FUNC_CMPSET(int, u_int);
1297 MSAN_ATOMIC_FUNC_CMPSET(long, u_long);
1298 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
1299
1300 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
1301 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
1302 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
1303 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
1304 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int);
1305 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long);
1306 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
1307
1308 _MSAN_ATOMIC_FUNC_LOAD(bool, bool);
1309 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t);
1310 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t);
1311 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t);
1312 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t);
1313 MSAN_ATOMIC_FUNC_LOAD(char, u_char);
1314 MSAN_ATOMIC_FUNC_LOAD(short, u_short);
1315 MSAN_ATOMIC_FUNC_LOAD(int, u_int);
1316 MSAN_ATOMIC_FUNC_LOAD(long, u_long);
1317 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
1318
1319 _MSAN_ATOMIC_FUNC_STORE(bool, bool);
1320 MSAN_ATOMIC_FUNC_STORE(8, uint8_t);
1321 MSAN_ATOMIC_FUNC_STORE(16, uint16_t);
1322 MSAN_ATOMIC_FUNC_STORE(32, uint32_t);
1323 MSAN_ATOMIC_FUNC_STORE(64, uint64_t);
1324 MSAN_ATOMIC_FUNC_STORE(char, u_char);
1325 MSAN_ATOMIC_FUNC_STORE(short, u_short);
1326 MSAN_ATOMIC_FUNC_STORE(int, u_int);
1327 MSAN_ATOMIC_FUNC_STORE(long, u_long);
1328 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
1329
1330 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq);
1331 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel);
1332 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
1333 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
1334
1335 void
1336 kmsan_atomic_interrupt_fence(void)
1337 {
1338         atomic_interrupt_fence();
1339 }
1340
1341 /* -------------------------------------------------------------------------- */
1342
1343 #include <sys/bus.h>
1344 #include <machine/bus.h>
1345 #include <sys/bus_san.h>
1346
1347 int
1348 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
1349     int flags, bus_space_handle_t *handlep)
1350 {
1351         return (bus_space_map(tag, hnd, size, flags, handlep));
1352 }
1353
1354 void
1355 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
1356     bus_size_t size)
1357 {
1358         bus_space_unmap(tag, hnd, size);
1359 }
1360
1361 int
1362 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
1363     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
1364 {
1365         return (bus_space_subregion(tag, hnd, offset, size, handlep));
1366 }
1367
1368 void
1369 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
1370     bus_size_t size)
1371 {
1372         bus_space_free(tag, hnd, size);
1373 }
1374
1375 void
1376 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
1377     bus_size_t offset, bus_size_t size, int flags)
1378 {
1379         bus_space_barrier(tag, hnd, offset, size, flags);
1380 }
1381
1382 /* XXXMJ x86-specific */
1383 #define MSAN_BUS_READ_FUNC(func, width, type)                           \
1384         type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
1385             bus_space_handle_t hnd, bus_size_t offset)                  \
1386         {                                                               \
1387                 type ret;                                               \
1388                 if ((tag) != X86_BUS_SPACE_IO)                          \
1389                         kmsan_shadow_fill((uintptr_t)(hnd + offset),    \
1390                             KMSAN_STATE_INITED, (width));               \
1391                 ret = bus_space_read##func##_##width(tag, hnd, offset); \
1392                 kmsan_init_ret(sizeof(type));                           \
1393                 return (ret);                                           \
1394         }                                                               \
1395
1396 #define MSAN_BUS_READ_PTR_FUNC(func, width, type)                       \
1397         void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
1398             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
1399             bus_size_t count)                                           \
1400         {                                                               \
1401                 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED,   \
1402                     (width) * count);                                   \
1403                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
1404                     count);                                             \
1405         }
1406
1407 MSAN_BUS_READ_FUNC(, 1, uint8_t)
1408 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
1409 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
1410 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
1411 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
1412 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
1413
1414 MSAN_BUS_READ_FUNC(, 2, uint16_t)
1415 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
1416 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
1417 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
1418 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
1419 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
1420
1421 MSAN_BUS_READ_FUNC(, 4, uint32_t)
1422 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
1423 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
1424 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
1425 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
1426 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
1427
1428 MSAN_BUS_READ_FUNC(, 8, uint64_t)
1429
1430 #define MSAN_BUS_WRITE_FUNC(func, width, type)                          \
1431         void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
1432             bus_space_handle_t hnd, bus_size_t offset, type value)      \
1433         {                                                               \
1434                 bus_space_write##func##_##width(tag, hnd, offset, value);\
1435         }                                                               \
1436
1437 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
1438         void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
1439             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
1440             bus_size_t count)                                           \
1441         {                                                               \
1442                 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
1443                     "bus_space_write()");                               \
1444                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
1445                     count);                                             \
1446         }
1447
1448 MSAN_BUS_WRITE_FUNC(, 1, uint8_t)
1449 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
1450 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
1451 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
1452 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
1453 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
1454
1455 MSAN_BUS_WRITE_FUNC(, 2, uint16_t)
1456 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
1457 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
1458 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
1459 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1460 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1461
1462 MSAN_BUS_WRITE_FUNC(, 4, uint32_t)
1463 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1464 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1465 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1466 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1467 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1468
1469 MSAN_BUS_WRITE_FUNC(, 8, uint64_t)
1470
1471 #define MSAN_BUS_SET_FUNC(func, width, type)                            \
1472         void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
1473             bus_space_handle_t hnd, bus_size_t offset, type value,      \
1474             bus_size_t count)                                           \
1475         {                                                               \
1476                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1477                     count);                                             \
1478         }
1479
1480 MSAN_BUS_SET_FUNC(multi, 1, uint8_t)
1481 MSAN_BUS_SET_FUNC(region, 1, uint8_t)
1482 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1483 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1484
1485 MSAN_BUS_SET_FUNC(multi, 2, uint16_t)
1486 MSAN_BUS_SET_FUNC(region, 2, uint16_t)
1487 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1488 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1489
1490 MSAN_BUS_SET_FUNC(multi, 4, uint32_t)
1491 MSAN_BUS_SET_FUNC(region, 4, uint32_t)
1492 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1493 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1494
1495 /* -------------------------------------------------------------------------- */
1496
1497 void
1498 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op)
1499 {
1500         /*
1501          * Some drivers, e.g., nvme, use the same code path for loading device
1502          * read and write requests, and will thus specify both flags.  In this
1503          * case we should not do any checking since it will generally lead to
1504          * false positives.
1505          */
1506         if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) ==
1507             BUS_DMASYNC_PREWRITE) {
1508                 switch (desc->md_type) {
1509                 case MEMDESC_VADDR:
1510                         kmsan_check(desc->u.md_vaddr, desc->md_len,
1511                             "dmasync");
1512                         break;
1513                 case MEMDESC_MBUF:
1514                         kmsan_check_mbuf(desc->u.md_mbuf, "dmasync");
1515                         break;
1516                 case 0:
1517                         break;
1518                 default:
1519                         kmsan_panic("%s: unhandled memdesc type %d", __func__,
1520                             desc->md_type);
1521                 }
1522         }
1523         if ((op & BUS_DMASYNC_POSTREAD) != 0) {
1524                 switch (desc->md_type) {
1525                 case MEMDESC_VADDR:
1526                         kmsan_mark(desc->u.md_vaddr, desc->md_len,
1527                             KMSAN_STATE_INITED);
1528                         break;
1529                 case MEMDESC_MBUF:
1530                         kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED);
1531                         break;
1532                 case 0:
1533                         break;
1534                 default:
1535                         kmsan_panic("%s: unhandled memdesc type %d", __func__,
1536                             desc->md_type);
1537                 }
1538         }
1539 }