sys/kern/subr_msan.c

   1 /*      $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $      */
   2
   3 /*
   4  * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
   5  * All rights reserved.
   6  * Copyright (c) 2021 The FreeBSD Foundation
   7  *
   8  * Portions of this software were developed by Mark Johnston under sponsorship
   9  * from the FreeBSD Foundation.
  10  *
  11  * This code is part of the KMSAN subsystem of the NetBSD kernel.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  23  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  24  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  25  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 #define SAN_RUNTIME
  36
  37 #include <sys/cdefs.h>
  38 __FBSDID("$FreeBSD$");
  39 #if 0
  40 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $");
  41 #endif
  42
  43 #include <sys/param.h>
  44 #include <sys/systm.h>
  45 #include <sys/bio.h>
  46 #include <sys/buf.h>
  47 #include <sys/conf.h>
  48 #include <sys/kdb.h>
  49 #include <sys/kernel.h>
  50 #include <sys/linker.h>
  51 #include <sys/malloc.h>
  52 #include <sys/mbuf.h>
  53 #include <sys/memdesc.h>
  54 #include <sys/msan.h>
  55 #include <sys/proc.h>
  56 #include <sys/stack.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/uio.h>
  59
  60 #include <vm/vm.h>
  61 #include <vm/pmap.h>
  62
  63 #include <machine/msan.h>
  64 #include <machine/stdarg.h>
  65
  66 void kmsan_init_arg(size_t);
  67 void kmsan_init_ret(size_t);
  68
  69 /* -------------------------------------------------------------------------- */
  70
  71 /*
  72  * Part of the compiler ABI.
  73  */
  74
  75 typedef struct {
  76         uint8_t *shad;
  77         msan_orig_t *orig;
  78 } msan_meta_t;
  79
  80 #define MSAN_PARAM_SIZE         800
  81 #define MSAN_RETVAL_SIZE        800
  82 typedef struct {
  83         uint8_t param_shadow[MSAN_PARAM_SIZE];
  84         uint8_t retval_shadow[MSAN_RETVAL_SIZE];
  85         uint8_t va_arg_shadow[MSAN_PARAM_SIZE];
  86         uint8_t va_arg_origin[MSAN_PARAM_SIZE];
  87         uint64_t va_arg_overflow_size;
  88         msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)];
  89         msan_orig_t retval_origin;
  90 } msan_tls_t;
  91
  92 /* -------------------------------------------------------------------------- */
  93
  94 #define MSAN_NCONTEXT   4
  95 #define MSAN_ORIG_MASK  (~0x3)
  96
  97 typedef struct kmsan_td {
  98         size_t ctx;
  99         msan_tls_t tls[MSAN_NCONTEXT];
 100 } msan_td_t;
 101
 102 static msan_tls_t dummy_tls;
 103
 104 /*
 105  * Use separate dummy regions for loads and stores: stores may mark the region
 106  * as uninitialized, and that can trigger false positives.
 107  */
 108 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
 109 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
 110 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
 111 static msan_td_t msan_thread0;
 112 static bool kmsan_enabled __read_mostly;
 113
 114 static bool kmsan_reporting = false;
 115
 116 /*
 117  * Avoid clobbering any thread-local state before we panic.
 118  */
 119 #define kmsan_panic(f, ...) do {                        \
 120         kmsan_enabled = false;                          \
 121         panic(f, __VA_ARGS__);                          \
 122 } while (0)
 123
 124 #define REPORT(f, ...) do {                             \
 125         if (panic_on_violation) {                       \
 126                 kmsan_panic(f, __VA_ARGS__);            \
 127         } else {                                        \
 128                 struct stack st;                        \
 129                                                         \
 130                 stack_save(&st);                        \
 131                 printf(f "\n", __VA_ARGS__);            \
 132                 stack_print_ddb(&st);                   \
 133         }                                               \
 134 } while (0)
 135
 136 FEATURE(kmsan, "Kernel memory sanitizer");
 137
 138 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 139     "KMSAN options");
 140
 141 static bool panic_on_violation = 1;
 142 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN,
 143     &panic_on_violation, 0,
 144     "Panic if an invalid access is detected");
 145
 146 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer");
 147
 148 /* -------------------------------------------------------------------------- */
 149
 150 static inline const char *
 151 kmsan_orig_name(int type)
 152 {
 153         switch (type) {
 154         case KMSAN_TYPE_STACK:
 155                 return ("stack");
 156         case KMSAN_TYPE_KMEM:
 157                 return ("kmem");
 158         case KMSAN_TYPE_MALLOC:
 159                 return ("malloc");
 160         case KMSAN_TYPE_UMA:
 161                 return ("UMA");
 162         default:
 163                 return ("unknown");
 164         }
 165 }
 166
 167 static void
 168 kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off,
 169     const char *hook)
 170 {
 171         const char *typename;
 172         char *var, *fn;
 173         uintptr_t ptr;
 174         long foff;
 175         char buf[128];
 176         int type;
 177
 178         if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
 179                 return;
 180
 181         kmsan_reporting = true;
 182         __compiler_membar();
 183
 184         if (*orig == 0) {
 185                 REPORT("MSan: Uninitialized memory in %s, offset %zu",
 186                     hook, off);
 187                 goto out;
 188         }
 189
 190         kmsan_md_orig_decode(*orig, &type, &ptr);
 191         typename = kmsan_orig_name(type);
 192
 193         if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
 194             sizeof(buf), &foff) == 0) {
 195                 REPORT("MSan: Uninitialized %s memory in %s, "
 196                     "offset %zu/%zu, addr %p, from %s+%#lx",
 197                     typename, hook, off, size, addr, buf, foff);
 198         } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
 199                 /*
 200                  * The format of the string is: "----var@function". Parse it to
 201                  * display a nice warning.
 202                  */
 203                 var = (char *)ptr + 4;
 204                 strlcpy(buf, var, sizeof(buf));
 205                 var = buf;
 206                 fn = strchr(buf, '@');
 207                 *fn++ = '\0';
 208                 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, "
 209                     "variable '%s' from %s", typename, hook, off, var, fn);
 210         } else {
 211                 REPORT("MSan: Uninitialized %s memory in %s, "
 212                     "offset %zu/%zu, addr %p, PC %p",
 213                     typename, hook, off, size, addr, (void *)ptr);
 214         }
 215
 216 out:
 217         __compiler_membar();
 218         kmsan_reporting = false;
 219 }
 220
 221 static void
 222 kmsan_report_inline(msan_orig_t orig, unsigned long pc)
 223 {
 224         const char *typename;
 225         char *var, *fn;
 226         uintptr_t ptr;
 227         char buf[128];
 228         long foff;
 229         int type;
 230
 231         if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
 232                 return;
 233
 234         kmsan_reporting = true;
 235         __compiler_membar();
 236
 237         if (orig == 0) {
 238                 REPORT("MSan: uninitialized variable in %p", (void *)pc);
 239                 goto out;
 240         }
 241
 242         kmsan_md_orig_decode(orig, &type, &ptr);
 243         typename = kmsan_orig_name(type);
 244
 245         if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
 246             sizeof(buf), &foff) == 0) {
 247                 REPORT("MSan: Uninitialized %s memory from %s+%#lx",
 248                     typename, buf, foff);
 249         } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
 250                 /*
 251                  * The format of the string is: "----var@function". Parse it to
 252                  * display a nice warning.
 253                  */
 254                 var = (char *)ptr + 4;
 255                 strlcpy(buf, var, sizeof(buf));
 256                 var = buf;
 257                 fn = strchr(buf, '@');
 258                 *fn++ = '\0';
 259                 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn);
 260         } else {
 261                 REPORT("MSan: Uninitialized %s memory, origin %x",
 262                     typename, orig);
 263         }
 264
 265 out:
 266         __compiler_membar();
 267         kmsan_reporting = false;
 268 }
 269
 270 /* -------------------------------------------------------------------------- */
 271
 272 static inline msan_meta_t
 273 kmsan_meta_get(const void *addr, size_t size, const bool write)
 274 {
 275         msan_meta_t ret;
 276
 277         if (__predict_false(!kmsan_enabled)) {
 278                 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
 279                 ret.orig = (msan_orig_t *)msan_dummy_orig;
 280         } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) {
 281                 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
 282                 ret.orig = (msan_orig_t *)msan_dummy_orig;
 283         } else {
 284                 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr);
 285                 ret.orig =
 286                     (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
 287                 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig &
 288                     MSAN_ORIG_MASK);
 289         }
 290
 291         return (ret);
 292 }
 293
 294 static inline void
 295 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size)
 296 {
 297         msan_orig_t *orig;
 298         size_t i;
 299
 300         if (__predict_false(!kmsan_enabled))
 301                 return;
 302         if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr)))
 303                 return;
 304
 305         orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
 306         size += ((uintptr_t)orig & (sizeof(*orig) - 1));
 307         orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
 308
 309         for (i = 0; i < size; i += 4) {
 310                 orig[i / 4] = o;
 311         }
 312 }
 313
 314 static inline void
 315 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size)
 316 {
 317         uint8_t *shad;
 318
 319         if (__predict_false(!kmsan_enabled))
 320                 return;
 321         if (__predict_false(kmsan_md_unsupported(addr)))
 322                 return;
 323
 324         shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
 325         __builtin_memset(shad, c, size);
 326 }
 327
 328 static inline void
 329 kmsan_meta_copy(void *dst, const void *src, size_t size)
 330 {
 331         uint8_t *orig_src, *orig_dst;
 332         uint8_t *shad_src, *shad_dst;
 333         msan_orig_t *_src, *_dst;
 334         size_t i;
 335
 336         if (__predict_false(!kmsan_enabled))
 337                 return;
 338         if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst)))
 339                 return;
 340         if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) {
 341                 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size);
 342                 return;
 343         }
 344
 345         shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src);
 346         shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst);
 347         __builtin_memmove(shad_dst, shad_src, size);
 348
 349         orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src);
 350         orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst);
 351         for (i = 0; i < size; i++) {
 352                 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK);
 353                 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK);
 354                 *_dst = *_src;
 355                 orig_src++;
 356                 orig_dst++;
 357         }
 358 }
 359
 360 static inline void
 361 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook)
 362 {
 363         msan_orig_t *orig;
 364         uint8_t *shad;
 365         size_t i;
 366
 367         if (__predict_false(!kmsan_enabled))
 368                 return;
 369         if (__predict_false(kmsan_md_unsupported(addr)))
 370                 return;
 371
 372         shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
 373         for (i = 0; i < size; i++) {
 374                 if (__predict_true(shad[i] == 0))
 375                         continue;
 376                 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)&shad[i]);
 377                 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
 378                 kmsan_report_hook((const char *)addr + i, orig, size, i, hook);
 379                 break;
 380         }
 381 }
 382
 383 void
 384 kmsan_init_arg(size_t n)
 385 {
 386         msan_td_t *mtd;
 387         uint8_t *arg;
 388
 389         if (__predict_false(!kmsan_enabled))
 390                 return;
 391         if (__predict_false(curthread == NULL))
 392                 return;
 393         mtd = curthread->td_kmsan;
 394         arg = mtd->tls[mtd->ctx].param_shadow;
 395         __builtin_memset(arg, 0, n);
 396 }
 397
 398 void
 399 kmsan_init_ret(size_t n)
 400 {
 401         msan_td_t *mtd;
 402         uint8_t *arg;
 403
 404         if (__predict_false(!kmsan_enabled))
 405                 return;
 406         if (__predict_false(curthread == NULL))
 407                 return;
 408         mtd = curthread->td_kmsan;
 409         arg = mtd->tls[mtd->ctx].retval_shadow;
 410         __builtin_memset(arg, 0, n);
 411 }
 412
 413 static void
 414 kmsan_check_arg(size_t size, const char *hook)
 415 {
 416         msan_orig_t *orig;
 417         msan_td_t *mtd;
 418         uint8_t *arg;
 419         size_t ctx, i;
 420
 421         if (__predict_false(!kmsan_enabled))
 422                 return;
 423         if (__predict_false(curthread == NULL))
 424                 return;
 425         mtd = curthread->td_kmsan;
 426         ctx = mtd->ctx;
 427         arg = mtd->tls[ctx].param_shadow;
 428
 429         for (i = 0; i < size; i++) {
 430                 if (__predict_true(arg[i] == 0))
 431                         continue;
 432                 orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)];
 433                 kmsan_report_hook((const char *)arg + i, orig, size, i, hook);
 434                 break;
 435         }
 436 }
 437
 438 void
 439 kmsan_thread_alloc(struct thread *td)
 440 {
 441         msan_td_t *mtd;
 442
 443         if (!kmsan_enabled)
 444                 return;
 445
 446         mtd = td->td_kmsan;
 447         if (mtd == NULL) {
 448                 /* We might be recycling a thread. */
 449                 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) +
 450                     sizeof(int));
 451                 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK);
 452         }
 453         kmsan_memset(mtd, 0, sizeof(*mtd));
 454         mtd->ctx = 0;
 455
 456         if (td->td_kstack != 0)
 457                 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages),
 458                     KMSAN_STATE_UNINIT);
 459
 460         td->td_kmsan = mtd;
 461 }
 462
 463 void
 464 kmsan_thread_free(struct thread *td)
 465 {
 466         msan_td_t *mtd;
 467
 468         if (!kmsan_enabled)
 469                 return;
 470         if (__predict_false(td == curthread))
 471                 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__);
 472
 473         mtd = td->td_kmsan;
 474         kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *));
 475         free(mtd, M_KMSAN);
 476         td->td_kmsan = NULL;
 477 }
 478
 479 void kmsan_intr_enter(void);
 480 void kmsan_intr_leave(void);
 481
 482 void
 483 kmsan_intr_enter(void)
 484 {
 485         msan_td_t *mtd;
 486
 487         if (__predict_false(!kmsan_enabled))
 488                 return;
 489
 490         mtd = curthread->td_kmsan;
 491         mtd->ctx++;
 492         if (__predict_false(mtd->ctx >= MSAN_NCONTEXT))
 493                 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
 494 }
 495
 496 void
 497 kmsan_intr_leave(void)
 498 {
 499         msan_td_t *mtd;
 500
 501         if (__predict_false(!kmsan_enabled))
 502                 return;
 503
 504         mtd = curthread->td_kmsan;
 505         if (__predict_false(mtd->ctx == 0))
 506                 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
 507         mtd->ctx--;
 508 }
 509
 510 /* -------------------------------------------------------------------------- */
 511
 512 void
 513 kmsan_shadow_map(vm_offset_t addr, size_t size)
 514 {
 515         size_t npages, i;
 516         vm_offset_t va;
 517
 518         MPASS(addr % PAGE_SIZE == 0);
 519         MPASS(size % PAGE_SIZE == 0);
 520
 521         if (!kmsan_enabled)
 522                 return;
 523
 524         npages = atop(size);
 525
 526         va = kmsan_md_addr_to_shad(addr);
 527         for (i = 0; i < npages; i++) {
 528                 pmap_san_enter(va + ptoa(i));
 529         }
 530
 531         va = kmsan_md_addr_to_orig(addr);
 532         for (i = 0; i < npages; i++) {
 533                 pmap_san_enter(va + ptoa(i));
 534         }
 535 }
 536
 537 void
 538 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc)
 539 {
 540         msan_orig_t orig;
 541
 542         orig = kmsan_md_orig_encode(type, pc);
 543         kmsan_origin_fill(addr, orig, size);
 544 }
 545
 546 void
 547 kmsan_mark(const void *addr, size_t size, uint8_t c)
 548 {
 549         kmsan_shadow_fill((uintptr_t)addr, c, size);
 550 }
 551
 552 void
 553 kmsan_mark_bio(const struct bio *bp, uint8_t c)
 554 {
 555         kmsan_mark(bp->bio_data, bp->bio_length, c);
 556 }
 557
 558 void
 559 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c)
 560 {
 561         do {
 562                 if ((m->m_flags & M_EXTPG) == 0)
 563                         kmsan_mark(m->m_data, m->m_len, c);
 564                 m = m->m_next;
 565         } while (m != NULL);
 566 }
 567
 568 void
 569 kmsan_check(const void *p, size_t sz, const char *descr)
 570 {
 571         kmsan_shadow_check((uintptr_t)p, sz, descr);
 572 }
 573
 574 void
 575 kmsan_check_bio(const struct bio *bp, const char *descr)
 576 {
 577         kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr);
 578 }
 579
 580 void
 581 kmsan_check_mbuf(const struct mbuf *m, const char *descr)
 582 {
 583         do {
 584                 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr);
 585         } while ((m = m->m_next) != NULL);
 586 }
 587
 588 void
 589 kmsan_init(void)
 590 {
 591         int disabled;
 592
 593         disabled = 0;
 594         TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled);
 595         if (disabled)
 596                 return;
 597
 598         /* Initialize the TLS for curthread. */
 599         msan_thread0.ctx = 0;
 600         thread0.td_kmsan = &msan_thread0;
 601
 602         /* Now officially enabled. */
 603         kmsan_enabled = true;
 604 }
 605
 606 /* -------------------------------------------------------------------------- */
 607
 608 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
 609 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
 610
 611 msan_meta_t
 612 __msan_metadata_ptr_for_load_n(void *addr, size_t size)
 613 {
 614         return (kmsan_meta_get(addr, size, false));
 615 }
 616
 617 msan_meta_t
 618 __msan_metadata_ptr_for_store_n(void *addr, size_t size)
 619 {
 620         return (kmsan_meta_get(addr, size, true));
 621 }
 622
 623 #define MSAN_META_FUNC(size)                                            \
 624         msan_meta_t __msan_metadata_ptr_for_load_##size(void *);        \
 625         msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr)     \
 626         {                                                               \
 627                 return (kmsan_meta_get(addr, size, false));             \
 628         }                                                               \
 629         msan_meta_t __msan_metadata_ptr_for_store_##size(void *);       \
 630         msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr)    \
 631         {                                                               \
 632                 return (kmsan_meta_get(addr, size, true));              \
 633         }
 634
 635 MSAN_META_FUNC(1)
 636 MSAN_META_FUNC(2)
 637 MSAN_META_FUNC(4)
 638 MSAN_META_FUNC(8)
 639
 640 void __msan_instrument_asm_store(const void *, size_t);
 641 msan_orig_t __msan_chain_origin(msan_orig_t);
 642 void __msan_poison(const void *, size_t);
 643 void __msan_unpoison(const void *, size_t);
 644 void __msan_poison_alloca(const void *, uint64_t, const char *);
 645 void __msan_unpoison_alloca(const void *, uint64_t);
 646 void __msan_warning(msan_orig_t);
 647 msan_tls_t *__msan_get_context_state(void);
 648
 649 void
 650 __msan_instrument_asm_store(const void *addr, size_t size)
 651 {
 652         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
 653 }
 654
 655 msan_orig_t
 656 __msan_chain_origin(msan_orig_t origin)
 657 {
 658         return (origin);
 659 }
 660
 661 void
 662 __msan_poison(const void *addr, size_t size)
 663 {
 664         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
 665 }
 666
 667 void
 668 __msan_unpoison(const void *addr, size_t size)
 669 {
 670         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
 671 }
 672
 673 void
 674 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr)
 675 {
 676         msan_orig_t orig;
 677
 678         orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
 679         kmsan_origin_fill(addr, orig, size);
 680         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
 681 }
 682
 683 void
 684 __msan_unpoison_alloca(const void *addr, uint64_t size)
 685 {
 686         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
 687 }
 688
 689 void
 690 __msan_warning(msan_orig_t origin)
 691 {
 692         if (__predict_false(!kmsan_enabled))
 693                 return;
 694         kmsan_report_inline(origin, KMSAN_RET_ADDR);
 695 }
 696
 697 msan_tls_t *
 698 __msan_get_context_state(void)
 699 {
 700         msan_td_t *mtd;
 701
 702         /*
 703          * When APs are started, they execute some C code before curthread is
 704          * set.  We have to handle that here.
 705          */
 706         if (__predict_false(!kmsan_enabled || curthread == NULL))
 707                 return (&dummy_tls);
 708         mtd = curthread->td_kmsan;
 709         return (&mtd->tls[mtd->ctx]);
 710 }
 711
 712 /* -------------------------------------------------------------------------- */
 713
 714 /*
 715  * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
 716  * initialized areas properly.
 717  */
 718
 719 void *
 720 kmsan_memcpy(void *dst, const void *src, size_t len)
 721 {
 722         /* No kmsan_check_arg, because inlined. */
 723         kmsan_init_ret(sizeof(void *));
 724         if (__predict_true(len != 0)) {
 725                 kmsan_meta_copy(dst, src, len);
 726         }
 727         return (__builtin_memcpy(dst, src, len));
 728 }
 729
 730 int
 731 kmsan_memcmp(const void *b1, const void *b2, size_t len)
 732 {
 733         const uint8_t *_b1 = b1, *_b2 = b2;
 734         size_t i;
 735
 736         kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len),
 737             "memcmp():args");
 738         kmsan_init_ret(sizeof(int));
 739
 740         for (i = 0; i < len; i++) {
 741                 if (*_b1 != *_b2) {
 742                         kmsan_shadow_check((uintptr_t)b1, i + 1,
 743                             "memcmp():arg1");
 744                         kmsan_shadow_check((uintptr_t)b2, i + 1,
 745                             "memcmp():arg2");
 746                         return (*_b1 - *_b2);
 747                 }
 748                 _b1++, _b2++;
 749         }
 750
 751         return (0);
 752 }
 753
 754 void *
 755 kmsan_memset(void *dst, int c, size_t len)
 756 {
 757         /* No kmsan_check_arg, because inlined. */
 758         kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len);
 759         kmsan_init_ret(sizeof(void *));
 760         return (__builtin_memset(dst, c, len));
 761 }
 762
 763 void *
 764 kmsan_memmove(void *dst, const void *src, size_t len)
 765 {
 766         /* No kmsan_check_arg, because inlined. */
 767         if (__predict_true(len != 0)) {
 768                 kmsan_meta_copy(dst, src, len);
 769         }
 770         kmsan_init_ret(sizeof(void *));
 771         return (__builtin_memmove(dst, src, len));
 772 }
 773
 774 __strong_reference(kmsan_memcpy, __msan_memcpy);
 775 __strong_reference(kmsan_memset, __msan_memset);
 776 __strong_reference(kmsan_memmove, __msan_memmove);
 777
 778 char *
 779 kmsan_strcpy(char *dst, const char *src)
 780 {
 781         const char *_src = src;
 782         char *_dst = dst;
 783         size_t len = 0;
 784
 785         kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args");
 786
 787         while (1) {
 788                 len++;
 789                 *dst = *src;
 790                 if (*src == '\0')
 791                         break;
 792                 src++, dst++;
 793         }
 794
 795         kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2");
 796         kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len);
 797         kmsan_init_ret(sizeof(char *));
 798         return (_dst);
 799 }
 800
 801 int
 802 kmsan_strcmp(const char *s1, const char *s2)
 803 {
 804         const char *_s1 = s1, *_s2 = s2;
 805         size_t len = 0;
 806
 807         kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args");
 808         kmsan_init_ret(sizeof(int));
 809
 810         while (1) {
 811                 len++;
 812                 if (*s1 != *s2)
 813                         break;
 814                 if (*s1 == '\0') {
 815                         kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
 816                         kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
 817                         return (0);
 818                 }
 819                 s1++, s2++;
 820         }
 821
 822         kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
 823         kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
 824
 825         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 826 }
 827
 828 size_t
 829 kmsan_strlen(const char *str)
 830 {
 831         const char *s;
 832
 833         kmsan_check_arg(sizeof(str), "strlen():args");
 834
 835         s = str;
 836         while (1) {
 837                 if (*s == '\0')
 838                         break;
 839                 s++;
 840         }
 841
 842         kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1");
 843         kmsan_init_ret(sizeof(size_t));
 844         return (s - str);
 845 }
 846
 847 int     kmsan_copyin(const void *, void *, size_t);
 848 int     kmsan_copyout(const void *, void *, size_t);
 849 int     kmsan_copyinstr(const void *, void *, size_t, size_t *);
 850
 851 int
 852 kmsan_copyin(const void *uaddr, void *kaddr, size_t len)
 853 {
 854         int ret;
 855
 856         kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len),
 857             "copyin():args");
 858         ret = copyin(uaddr, kaddr, len);
 859         if (ret == 0)
 860                 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len);
 861         kmsan_init_ret(sizeof(int));
 862         return (ret);
 863 }
 864
 865 int
 866 kmsan_copyout(const void *kaddr, void *uaddr, size_t len)
 867 {
 868         kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len),
 869             "copyout():args");
 870         kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1");
 871         kmsan_init_ret(sizeof(int));
 872         return (copyout(kaddr, uaddr, len));
 873 }
 874
 875 int
 876 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 877 {
 878         size_t _done;
 879         int ret;
 880
 881         kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) +
 882             sizeof(len) + sizeof(done), "copyinstr():args");
 883         ret = copyinstr(uaddr, kaddr, len, &_done);
 884         if (ret == 0)
 885                 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done);
 886         if (done != NULL) {
 887                 *done = _done;
 888                 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t));
 889         }
 890         kmsan_init_ret(sizeof(int));
 891         return (ret);
 892 }
 893
 894 /* -------------------------------------------------------------------------- */
 895
 896 int
 897 kmsan_fubyte(volatile const void *base)
 898 {
 899         int ret;
 900
 901         kmsan_check_arg(sizeof(base), "fubyte(): args");
 902         ret = fubyte(base);
 903         kmsan_init_ret(sizeof(int));
 904         return (ret);
 905 }
 906
 907 int
 908 kmsan_fuword16(volatile const void *base)
 909 {
 910         int ret;
 911
 912         kmsan_check_arg(sizeof(base), "fuword16(): args");
 913         ret = fuword16(base);
 914         kmsan_init_ret(sizeof(int));
 915         return (ret);
 916 }
 917
 918 int
 919 kmsan_fueword(volatile const void *base, long *val)
 920 {
 921         int ret;
 922
 923         kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args");
 924         ret = fueword(base, val);
 925         if (ret == 0)
 926                 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
 927                     sizeof(*val));
 928         kmsan_init_ret(sizeof(int));
 929         return (ret);
 930 }
 931
 932 int
 933 kmsan_fueword32(volatile const void *base, int32_t *val)
 934 {
 935         int ret;
 936
 937         kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args");
 938         ret = fueword32(base, val);
 939         if (ret == 0)
 940                 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
 941                     sizeof(*val));
 942         kmsan_init_ret(sizeof(int));
 943         return (ret);
 944 }
 945
 946 int
 947 kmsan_fueword64(volatile const void *base, int64_t *val)
 948 {
 949         int ret;
 950
 951         kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args");
 952         ret = fueword64(base, val);
 953         if (ret == 0)
 954                 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
 955                     sizeof(*val));
 956         kmsan_init_ret(sizeof(int));
 957         return (ret);
 958 }
 959
 960 int
 961 kmsan_subyte(volatile void *base, int byte)
 962 {
 963         int ret;
 964
 965         kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args");
 966         ret = subyte(base, byte);
 967         kmsan_init_ret(sizeof(int));
 968         return (ret);
 969 }
 970
 971 int
 972 kmsan_suword(volatile void *base, long word)
 973 {
 974         int ret;
 975
 976         kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args");
 977         ret = suword(base, word);
 978         kmsan_init_ret(sizeof(int));
 979         return (ret);
 980 }
 981
 982 int
 983 kmsan_suword16(volatile void *base, int word)
 984 {
 985         int ret;
 986
 987         kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args");
 988         ret = suword16(base, word);
 989         kmsan_init_ret(sizeof(int));
 990         return (ret);
 991 }
 992
 993 int
 994 kmsan_suword32(volatile void *base, int32_t word)
 995 {
 996         int ret;
 997
 998         kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args");
 999         ret = suword32(base, word);
1000         kmsan_init_ret(sizeof(int));
1001         return (ret);
1002 }
1003
1004 int
1005 kmsan_suword64(volatile void *base, int64_t word)
1006 {
1007         int ret;
1008
1009         kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args");
1010         ret = suword64(base, word);
1011         kmsan_init_ret(sizeof(int));
1012         return (ret);
1013 }
1014
1015 int
1016 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
1017     uint32_t newval)
1018 {
1019         int ret;
1020
1021         kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1022             sizeof(newval), "casueword32(): args");
1023         ret = casueword32(base, oldval, oldvalp, newval);
1024         kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1025             sizeof(*oldvalp));
1026         kmsan_init_ret(sizeof(int));
1027         return (ret);
1028 }
1029
1030 int
1031 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
1032     u_long newval)
1033 {
1034         int ret;
1035
1036         kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1037             sizeof(newval), "casueword32(): args");
1038         ret = casueword(base, oldval, oldvalp, newval);
1039         kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1040             sizeof(*oldvalp));
1041         kmsan_init_ret(sizeof(int));
1042         return (ret);
1043 }
1044
1045 /* -------------------------------------------------------------------------- */
1046
1047 #include <machine/atomic.h>
1048 #include <sys/atomic_san.h>
1049
1050 #define _MSAN_ATOMIC_FUNC_ADD(name, type)                               \
1051         void kmsan_atomic_add_##name(volatile type *ptr, type val)      \
1052         {                                                               \
1053                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1054                     "atomic_add_" #name "():args");                     \
1055                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1056                     "atomic_add_" #name "():ptr");                      \
1057                 atomic_add_##name(ptr, val);                            \
1058         }
1059
1060 #define MSAN_ATOMIC_FUNC_ADD(name, type)                                \
1061         _MSAN_ATOMIC_FUNC_ADD(name, type)                               \
1062         _MSAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
1063         _MSAN_ATOMIC_FUNC_ADD(rel_##name, type)
1064
1065 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
1066         void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \
1067         {                                                               \
1068                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1069                     "atomic_subtract_" #name "():args");                \
1070                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1071                     "atomic_subtract_" #name "():ptr");                 \
1072                 atomic_subtract_##name(ptr, val);                       \
1073         }
1074
1075 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
1076         _MSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
1077         _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
1078         _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
1079
1080 #define _MSAN_ATOMIC_FUNC_SET(name, type)                               \
1081         void kmsan_atomic_set_##name(volatile type *ptr, type val)      \
1082         {                                                               \
1083                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1084                     "atomic_set_" #name "():args");                     \
1085                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1086                     "atomic_set_" #name "():ptr");                      \
1087                 atomic_set_##name(ptr, val);                            \
1088         }
1089
1090 #define MSAN_ATOMIC_FUNC_SET(name, type)                                \
1091         _MSAN_ATOMIC_FUNC_SET(name, type)                               \
1092         _MSAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
1093         _MSAN_ATOMIC_FUNC_SET(rel_##name, type)
1094
1095 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
1096         void kmsan_atomic_clear_##name(volatile type *ptr, type val)    \
1097         {                                                               \
1098                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1099                     "atomic_clear_" #name "():args");                   \
1100                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1101                     "atomic_clear_" #name "():ptr");                    \
1102                 atomic_clear_##name(ptr, val);                          \
1103         }
1104
1105 #define MSAN_ATOMIC_FUNC_CLEAR(name, type)                              \
1106         _MSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
1107         _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
1108         _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
1109
1110 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
1111         type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
1112         {                                                               \
1113                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1114                     "atomic_fetchadd_" #name "():args");                \
1115                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1116                     "atomic_fetchadd_" #name "():ptr");                 \
1117                 kmsan_init_ret(sizeof(type));                           \
1118                 return (atomic_fetchadd_##name(ptr, val));              \
1119         }
1120
1121 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
1122         type kmsan_atomic_readandclear_##name(volatile type *ptr)       \
1123         {                                                               \
1124                 kmsan_check_arg(sizeof(ptr),                            \
1125                     "atomic_readandclear_" #name "():args");            \
1126                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1127                     "atomic_readandclear_" #name "():ptr");             \
1128                 kmsan_init_ret(sizeof(type));                           \
1129                 return (atomic_readandclear_##name(ptr));               \
1130         }
1131
1132 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
1133         int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
1134         {                                                               \
1135                 kmsan_check_arg(sizeof(ptr) + sizeof(v),                \
1136                     "atomic_testandclear_" #name "():args");            \
1137                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1138                     "atomic_testandclear_" #name "():ptr");             \
1139                 kmsan_init_ret(sizeof(int));                            \
1140                 return (atomic_testandclear_##name(ptr, v));            \
1141         }
1142
1143 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
1144         int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \
1145         {                                                               \
1146                 kmsan_check_arg(sizeof(ptr) + sizeof(v),                \
1147                     "atomic_testandset_" #name "():args");              \
1148                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1149                     "atomic_testandset_" #name "():ptr");               \
1150                 kmsan_init_ret(sizeof(int));                            \
1151                 return (atomic_testandset_##name(ptr, v));              \
1152         }
1153
1154 #define MSAN_ATOMIC_FUNC_SWAP(name, type)                               \
1155         type kmsan_atomic_swap_##name(volatile type *ptr, type val)     \
1156         {                                                               \
1157                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1158                     "atomic_swap_" #name "():args");                    \
1159                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1160                     "atomic_swap_" #name "():ptr");                     \
1161                 kmsan_init_ret(sizeof(type));                           \
1162                 return (atomic_swap_##name(ptr, val));                  \
1163         }
1164
1165 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
1166         int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval,   \
1167             type nval)                                                  \
1168         {                                                               \
1169                 kmsan_check_arg(sizeof(ptr) + sizeof(oval) +            \
1170                     sizeof(nval), "atomic_cmpset_" #name "():args");    \
1171                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1172                     "atomic_cmpset_" #name "():ptr");                   \
1173                 kmsan_init_ret(sizeof(int));                            \
1174                 return (atomic_cmpset_##name(ptr, oval, nval));         \
1175         }
1176
1177 #define MSAN_ATOMIC_FUNC_CMPSET(name, type)                             \
1178         _MSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
1179         _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
1180         _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
1181
1182 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
1183         int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
1184             type nval)                                                  \
1185         {                                                               \
1186                 kmsan_check_arg(sizeof(ptr) + sizeof(oval) +            \
1187                     sizeof(nval), "atomic_fcmpset_" #name "():args");   \
1188                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1189                     "atomic_fcmpset_" #name "():ptr");                  \
1190                 kmsan_init_ret(sizeof(int));                            \
1191                 return (atomic_fcmpset_##name(ptr, oval, nval));        \
1192         }
1193
1194 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
1195         _MSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
1196         _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
1197         _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
1198
1199 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
1200         void kmsan_atomic_thread_fence_##name(void)                     \
1201         {                                                               \
1202                 atomic_thread_fence_##name();                           \
1203         }
1204
1205 #define _MSAN_ATOMIC_FUNC_LOAD(name, type)                              \
1206         type kmsan_atomic_load_##name(volatile type *ptr)               \
1207         {                                                               \
1208                 kmsan_check_arg(sizeof(ptr),                            \
1209                     "atomic_load_" #name "():args");                    \
1210                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1211                     "atomic_load_" #name "():ptr");                     \
1212                 kmsan_init_ret(sizeof(type));                           \
1213                 return (atomic_load_##name(ptr));                       \
1214         }
1215
1216 #define MSAN_ATOMIC_FUNC_LOAD(name, type)                               \
1217         _MSAN_ATOMIC_FUNC_LOAD(name, type)                              \
1218         _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type)
1219
1220 #define _MSAN_ATOMIC_FUNC_STORE(name, type)                             \
1221         void kmsan_atomic_store_##name(volatile type *ptr, type val)    \
1222         {                                                               \
1223                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1224                     "atomic_store_" #name "():args");                   \
1225                 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED,   \
1226                     sizeof(type));                                      \
1227                 atomic_store_##name(ptr, val);                          \
1228         }
1229
1230 #define MSAN_ATOMIC_FUNC_STORE(name, type)                              \
1231         _MSAN_ATOMIC_FUNC_STORE(name, type)                             \
1232         _MSAN_ATOMIC_FUNC_STORE(rel_##name, type)
1233
1234 MSAN_ATOMIC_FUNC_ADD(8, uint8_t);
1235 MSAN_ATOMIC_FUNC_ADD(16, uint16_t);
1236 MSAN_ATOMIC_FUNC_ADD(32, uint32_t);
1237 MSAN_ATOMIC_FUNC_ADD(64, uint64_t);
1238 MSAN_ATOMIC_FUNC_ADD(int, u_int);
1239 MSAN_ATOMIC_FUNC_ADD(long, u_long);
1240 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
1241
1242 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
1243 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
1244 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
1245 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
1246 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
1247 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
1248 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
1249
1250 MSAN_ATOMIC_FUNC_SET(8, uint8_t);
1251 MSAN_ATOMIC_FUNC_SET(16, uint16_t);
1252 MSAN_ATOMIC_FUNC_SET(32, uint32_t);
1253 MSAN_ATOMIC_FUNC_SET(64, uint64_t);
1254 MSAN_ATOMIC_FUNC_SET(int, u_int);
1255 MSAN_ATOMIC_FUNC_SET(long, u_long);
1256 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
1257
1258 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
1259 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
1260 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
1261 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
1262 MSAN_ATOMIC_FUNC_CLEAR(int, u_int);
1263 MSAN_ATOMIC_FUNC_CLEAR(long, u_long);
1264 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
1265
1266 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
1267 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
1268 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int);
1269 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long);
1270
1271 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
1272 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
1273 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
1274 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
1275 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
1276
1277 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
1278 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
1279 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
1280 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
1281
1282 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
1283 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
1284 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
1285 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
1286
1287 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t);
1288 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t);
1289 MSAN_ATOMIC_FUNC_SWAP(int, u_int);
1290 MSAN_ATOMIC_FUNC_SWAP(long, u_long);
1291 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
1292
1293 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
1294 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
1295 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
1296 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
1297 MSAN_ATOMIC_FUNC_CMPSET(int, u_int);
1298 MSAN_ATOMIC_FUNC_CMPSET(long, u_long);
1299 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
1300
1301 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
1302 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
1303 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
1304 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
1305 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int);
1306 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long);
1307 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
1308
1309 _MSAN_ATOMIC_FUNC_LOAD(bool, bool);
1310 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t);
1311 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t);
1312 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t);
1313 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t);
1314 MSAN_ATOMIC_FUNC_LOAD(char, u_char);
1315 MSAN_ATOMIC_FUNC_LOAD(short, u_short);
1316 MSAN_ATOMIC_FUNC_LOAD(int, u_int);
1317 MSAN_ATOMIC_FUNC_LOAD(long, u_long);
1318 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
1319
1320 _MSAN_ATOMIC_FUNC_STORE(bool, bool);
1321 MSAN_ATOMIC_FUNC_STORE(8, uint8_t);
1322 MSAN_ATOMIC_FUNC_STORE(16, uint16_t);
1323 MSAN_ATOMIC_FUNC_STORE(32, uint32_t);
1324 MSAN_ATOMIC_FUNC_STORE(64, uint64_t);
1325 MSAN_ATOMIC_FUNC_STORE(char, u_char);
1326 MSAN_ATOMIC_FUNC_STORE(short, u_short);
1327 MSAN_ATOMIC_FUNC_STORE(int, u_int);
1328 MSAN_ATOMIC_FUNC_STORE(long, u_long);
1329 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
1330
1331 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq);
1332 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel);
1333 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
1334 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
1335
1336 void
1337 kmsan_atomic_interrupt_fence(void)
1338 {
1339         atomic_interrupt_fence();
1340 }
1341
1342 /* -------------------------------------------------------------------------- */
1343
1344 #include <sys/bus.h>
1345 #include <machine/bus.h>
1346 #include <sys/bus_san.h>
1347
1348 int
1349 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
1350     int flags, bus_space_handle_t *handlep)
1351 {
1352         return (bus_space_map(tag, hnd, size, flags, handlep));
1353 }
1354
1355 void
1356 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
1357     bus_size_t size)
1358 {
1359         bus_space_unmap(tag, hnd, size);
1360 }
1361
1362 int
1363 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
1364     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
1365 {
1366         return (bus_space_subregion(tag, hnd, offset, size, handlep));
1367 }
1368
1369 void
1370 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
1371     bus_size_t size)
1372 {
1373         bus_space_free(tag, hnd, size);
1374 }
1375
1376 void
1377 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
1378     bus_size_t offset, bus_size_t size, int flags)
1379 {
1380         bus_space_barrier(tag, hnd, offset, size, flags);
1381 }
1382
1383 /* XXXMJ x86-specific */
1384 #define MSAN_BUS_READ_FUNC(func, width, type)                           \
1385         type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
1386             bus_space_handle_t hnd, bus_size_t offset)                  \
1387         {                                                               \
1388                 type ret;                                               \
1389                 if ((tag) != X86_BUS_SPACE_IO)                          \
1390                         kmsan_shadow_fill((uintptr_t)(hnd + offset),    \
1391                             KMSAN_STATE_INITED, (width));               \
1392                 ret = bus_space_read##func##_##width(tag, hnd, offset); \
1393                 kmsan_init_ret(sizeof(type));                           \
1394                 return (ret);                                           \
1395         }                                                               \
1396
1397 #define MSAN_BUS_READ_PTR_FUNC(func, width, type)                       \
1398         void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
1399             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
1400             bus_size_t count)                                           \
1401         {                                                               \
1402                 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED,   \
1403                     (width) * count);                                   \
1404                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
1405                     count);                                             \
1406         }
1407
1408 MSAN_BUS_READ_FUNC(, 1, uint8_t)
1409 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
1410 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
1411 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
1412 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
1413 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
1414
1415 MSAN_BUS_READ_FUNC(, 2, uint16_t)
1416 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
1417 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
1418 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
1419 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
1420 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
1421
1422 MSAN_BUS_READ_FUNC(, 4, uint32_t)
1423 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
1424 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
1425 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
1426 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
1427 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
1428
1429 MSAN_BUS_READ_FUNC(, 8, uint64_t)
1430
1431 #define MSAN_BUS_WRITE_FUNC(func, width, type)                          \
1432         void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
1433             bus_space_handle_t hnd, bus_size_t offset, type value)      \
1434         {                                                               \
1435                 bus_space_write##func##_##width(tag, hnd, offset, value);\
1436         }                                                               \
1437
1438 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
1439         void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
1440             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
1441             bus_size_t count)                                           \
1442         {                                                               \
1443                 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
1444                     "bus_space_write()");                               \
1445                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
1446                     count);                                             \
1447         }
1448
1449 MSAN_BUS_WRITE_FUNC(, 1, uint8_t)
1450 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
1451 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
1452 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
1453 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
1454 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
1455
1456 MSAN_BUS_WRITE_FUNC(, 2, uint16_t)
1457 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
1458 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
1459 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
1460 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1461 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1462
1463 MSAN_BUS_WRITE_FUNC(, 4, uint32_t)
1464 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1465 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1466 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1467 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1468 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1469
1470 MSAN_BUS_WRITE_FUNC(, 8, uint64_t)
1471
1472 #define MSAN_BUS_SET_FUNC(func, width, type)                            \
1473         void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
1474             bus_space_handle_t hnd, bus_size_t offset, type value,      \
1475             bus_size_t count)                                           \
1476         {                                                               \
1477                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1478                     count);                                             \
1479         }
1480
1481 MSAN_BUS_SET_FUNC(multi, 1, uint8_t)
1482 MSAN_BUS_SET_FUNC(region, 1, uint8_t)
1483 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1484 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1485
1486 MSAN_BUS_SET_FUNC(multi, 2, uint16_t)
1487 MSAN_BUS_SET_FUNC(region, 2, uint16_t)
1488 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1489 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1490
1491 MSAN_BUS_SET_FUNC(multi, 4, uint32_t)
1492 MSAN_BUS_SET_FUNC(region, 4, uint32_t)
1493 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1494 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1495
1496 /* -------------------------------------------------------------------------- */
1497
1498 void
1499 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op)
1500 {
1501         /*
1502          * Some drivers, e.g., nvme, use the same code path for loading device
1503          * read and write requests, and will thus specify both flags.  In this
1504          * case we should not do any checking since it will generally lead to
1505          * false positives.
1506          */
1507         if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) ==
1508             BUS_DMASYNC_PREWRITE) {
1509                 switch (desc->md_type) {
1510                 case MEMDESC_VADDR:
1511                         kmsan_check(desc->u.md_vaddr, desc->md_len,
1512                             "dmasync");
1513                         break;
1514                 case MEMDESC_MBUF:
1515                         kmsan_check_mbuf(desc->u.md_mbuf, "dmasync");
1516                         break;
1517                 case 0:
1518                         break;
1519                 default:
1520                         kmsan_panic("%s: unhandled memdesc type %d", __func__,
1521                             desc->md_type);
1522                 }
1523         }
1524         if ((op & BUS_DMASYNC_POSTREAD) != 0) {
1525                 switch (desc->md_type) {
1526                 case MEMDESC_VADDR:
1527                         kmsan_mark(desc->u.md_vaddr, desc->md_len,
1528                             KMSAN_STATE_INITED);
1529                         break;
1530                 case MEMDESC_MBUF:
1531                         kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED);
1532                         break;
1533                 case 0:
1534                         break;
1535                 default:
1536                         kmsan_panic("%s: unhandled memdesc type %d", __func__,
1537                             desc->md_type);
1538                 }
1539         }
1540 }