sys/kern/subr_msan.c

   1 /*      $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $      */
   2
   3 /*
   4  * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
   5  * All rights reserved.
   6  * Copyright (c) 2021 The FreeBSD Foundation
   7  *
   8  * Portions of this software were developed by Mark Johnston under sponsorship
   9  * from the FreeBSD Foundation.
  10  *
  11  * This code is part of the KMSAN subsystem of the NetBSD kernel.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  23  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  24  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  25  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 #define SAN_RUNTIME
  36
  37 #include <sys/cdefs.h>
  38 __FBSDID("$FreeBSD$");
  39 #if 0
  40 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $");
  41 #endif
  42
  43 #include <sys/param.h>
  44 #include <sys/systm.h>
  45 #include <sys/bio.h>
  46 #include <sys/buf.h>
  47 #include <sys/conf.h>
  48 #include <sys/kdb.h>
  49 #include <sys/kernel.h>
  50 #include <sys/linker.h>
  51 #include <sys/malloc.h>
  52 #include <sys/mbuf.h>
  53 #include <sys/memdesc.h>
  54 #include <sys/msan.h>
  55 #include <sys/proc.h>
  56 #include <sys/stack.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/uio.h>
  59
  60 #include <cam/cam.h>
  61 #include <cam/cam_ccb.h>
  62
  63 #include <vm/vm.h>
  64 #include <vm/pmap.h>
  65
  66 #include <machine/msan.h>
  67 #include <machine/stdarg.h>
  68
  69 void kmsan_init_arg(size_t);
  70 void kmsan_init_ret(size_t);
  71
  72 /* -------------------------------------------------------------------------- */
  73
  74 /*
  75  * Part of the compiler ABI.
  76  */
  77
  78 typedef struct {
  79         uint8_t *shad;
  80         msan_orig_t *orig;
  81 } msan_meta_t;
  82
  83 #define MSAN_PARAM_SIZE         800
  84 #define MSAN_RETVAL_SIZE        800
  85 typedef struct {
  86         uint8_t param_shadow[MSAN_PARAM_SIZE];
  87         uint8_t retval_shadow[MSAN_RETVAL_SIZE];
  88         uint8_t va_arg_shadow[MSAN_PARAM_SIZE];
  89         uint8_t va_arg_origin[MSAN_PARAM_SIZE];
  90         uint64_t va_arg_overflow_size;
  91         msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)];
  92         msan_orig_t retval_origin;
  93 } msan_tls_t;
  94
  95 /* -------------------------------------------------------------------------- */
  96
  97 #define MSAN_NCONTEXT   4
  98 #define MSAN_ORIG_MASK  (~0x3)
  99
 100 typedef struct kmsan_td {
 101         size_t ctx;
 102         msan_tls_t tls[MSAN_NCONTEXT];
 103 } msan_td_t;
 104
 105 static msan_tls_t dummy_tls;
 106
 107 /*
 108  * Use separate dummy regions for loads and stores: stores may mark the region
 109  * as uninitialized, and that can trigger false positives.
 110  */
 111 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
 112 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
 113 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
 114 static msan_td_t msan_thread0;
 115 static bool kmsan_enabled __read_mostly;
 116
 117 static bool kmsan_reporting = false;
 118
 119 /*
 120  * Avoid clobbering any thread-local state before we panic.
 121  */
 122 #define kmsan_panic(f, ...) do {                        \
 123         kmsan_enabled = false;                          \
 124         panic(f, __VA_ARGS__);                          \
 125 } while (0)
 126
 127 #define REPORT(f, ...) do {                             \
 128         if (panic_on_violation) {                       \
 129                 kmsan_panic(f, __VA_ARGS__);            \
 130         } else {                                        \
 131                 struct stack st;                        \
 132                                                         \
 133                 stack_save(&st);                        \
 134                 printf(f "\n", __VA_ARGS__);            \
 135                 stack_print_ddb(&st);                   \
 136         }                                               \
 137 } while (0)
 138
 139 FEATURE(kmsan, "Kernel memory sanitizer");
 140
 141 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 142     "KMSAN options");
 143
 144 static bool panic_on_violation = 1;
 145 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN,
 146     &panic_on_violation, 0,
 147     "Panic if an invalid access is detected");
 148
 149 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer");
 150
 151 /* -------------------------------------------------------------------------- */
 152
 153 static inline const char *
 154 kmsan_orig_name(int type)
 155 {
 156         switch (type) {
 157         case KMSAN_TYPE_STACK:
 158                 return ("stack");
 159         case KMSAN_TYPE_KMEM:
 160                 return ("kmem");
 161         case KMSAN_TYPE_MALLOC:
 162                 return ("malloc");
 163         case KMSAN_TYPE_UMA:
 164                 return ("UMA");
 165         default:
 166                 return ("unknown");
 167         }
 168 }
 169
 170 static void
 171 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook)
 172 {
 173         msan_orig_t *orig;
 174         const char *typename;
 175         char *var, *fn;
 176         uintptr_t ptr;
 177         long foff;
 178         char buf[128];
 179         int type;
 180
 181         if (__predict_false(panicstr != NULL || kdb_active || kmsan_reporting))
 182                 return;
 183
 184         kmsan_reporting = true;
 185         __compiler_membar();
 186
 187         orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
 188         orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
 189
 190         if (*orig == 0) {
 191                 REPORT("MSan: Uninitialized memory in %s, offset %zu",
 192                     hook, off);
 193                 goto out;
 194         }
 195
 196         kmsan_md_orig_decode(*orig, &type, &ptr);
 197         typename = kmsan_orig_name(type);
 198
 199         if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
 200             sizeof(buf), &foff) == 0) {
 201                 REPORT("MSan: Uninitialized %s memory in %s, "
 202                     "offset %zu/%zu, addr %p, from %s+%#lx",
 203                     typename, hook, off, size, addr, buf, foff);
 204         } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
 205                 /*
 206                  * The format of the string is: "----var@function". Parse it to
 207                  * display a nice warning.
 208                  */
 209                 var = (char *)ptr + 4;
 210                 strlcpy(buf, var, sizeof(buf));
 211                 var = buf;
 212                 fn = strchr(buf, '@');
 213                 *fn++ = '\0';
 214                 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, "
 215                     "variable '%s' from %s", typename, hook, off, var, fn);
 216         } else {
 217                 REPORT("MSan: Uninitialized %s memory in %s, "
 218                     "offset %zu/%zu, addr %p, PC %p",
 219                     typename, hook, off, size, addr, (void *)ptr);
 220         }
 221
 222 out:
 223         __compiler_membar();
 224         kmsan_reporting = false;
 225 }
 226
 227 static void
 228 kmsan_report_inline(msan_orig_t orig, unsigned long pc)
 229 {
 230         const char *typename;
 231         char *var, *fn;
 232         uintptr_t ptr;
 233         char buf[128];
 234         long foff;
 235         int type;
 236
 237         if (__predict_false(panicstr != NULL || kdb_active || kmsan_reporting))
 238                 return;
 239
 240         kmsan_reporting = true;
 241         __compiler_membar();
 242
 243         if (orig == 0) {
 244                 REPORT("MSan: uninitialized variable in %p", (void *)pc);
 245                 goto out;
 246         }
 247
 248         kmsan_md_orig_decode(orig, &type, &ptr);
 249         typename = kmsan_orig_name(type);
 250
 251         if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
 252             sizeof(buf), &foff) == 0) {
 253                 REPORT("MSan: Uninitialized %s memory from %s+%#lx",
 254                     typename, buf, foff);
 255         } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
 256                 /*
 257                  * The format of the string is: "----var@function". Parse it to
 258                  * display a nice warning.
 259                  */
 260                 var = (char *)ptr + 4;
 261                 strlcpy(buf, var, sizeof(buf));
 262                 var = buf;
 263                 fn = strchr(buf, '@');
 264                 *fn++ = '\0';
 265                 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn);
 266         } else {
 267                 REPORT("MSan: Uninitialized %s memory, origin %x",
 268                     typename, orig);
 269         }
 270
 271 out:
 272         __compiler_membar();
 273         kmsan_reporting = false;
 274 }
 275
 276 /* -------------------------------------------------------------------------- */
 277
 278 static inline msan_meta_t
 279 kmsan_meta_get(const void *addr, size_t size, const bool write)
 280 {
 281         msan_meta_t ret;
 282
 283         if (__predict_false(!kmsan_enabled)) {
 284                 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
 285                 ret.orig = (msan_orig_t *)msan_dummy_orig;
 286         } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) {
 287                 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
 288                 ret.orig = (msan_orig_t *)msan_dummy_orig;
 289         } else {
 290                 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr);
 291                 ret.orig =
 292                     (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
 293                 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig &
 294                     MSAN_ORIG_MASK);
 295         }
 296
 297         return (ret);
 298 }
 299
 300 static inline void
 301 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size)
 302 {
 303         msan_orig_t *orig;
 304         size_t i;
 305
 306         if (__predict_false(!kmsan_enabled))
 307                 return;
 308         if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr)))
 309                 return;
 310
 311         orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
 312         size += ((uintptr_t)orig & (sizeof(*orig) - 1));
 313         orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
 314
 315         for (i = 0; i < size; i += 4) {
 316                 orig[i / 4] = o;
 317         }
 318 }
 319
 320 static inline void
 321 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size)
 322 {
 323         uint8_t *shad;
 324
 325         if (__predict_false(!kmsan_enabled))
 326                 return;
 327         if (__predict_false(kmsan_md_unsupported(addr)))
 328                 return;
 329
 330         shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
 331         __builtin_memset(shad, c, size);
 332 }
 333
 334 static inline void
 335 kmsan_meta_copy(void *dst, const void *src, size_t size)
 336 {
 337         uint8_t *orig_src, *orig_dst;
 338         uint8_t *shad_src, *shad_dst;
 339         msan_orig_t *_src, *_dst;
 340         size_t i;
 341
 342         if (__predict_false(!kmsan_enabled))
 343                 return;
 344         if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst)))
 345                 return;
 346         if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) {
 347                 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size);
 348                 return;
 349         }
 350
 351         shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src);
 352         shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst);
 353         __builtin_memmove(shad_dst, shad_src, size);
 354
 355         orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src);
 356         orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst);
 357         for (i = 0; i < size; i++) {
 358                 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK);
 359                 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK);
 360                 *_dst = *_src;
 361                 orig_src++;
 362                 orig_dst++;
 363         }
 364 }
 365
 366 static inline void
 367 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook)
 368 {
 369         uint8_t *shad;
 370         size_t i;
 371
 372         if (__predict_false(!kmsan_enabled))
 373                 return;
 374         if (__predict_false(kmsan_md_unsupported(addr)))
 375                 return;
 376
 377         shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
 378         for (i = 0; i < size; i++) {
 379                 if (__predict_true(shad[i] == 0))
 380                         continue;
 381                 kmsan_report_hook((const char *)addr + i, size, i, hook);
 382                 break;
 383         }
 384 }
 385
 386 void
 387 kmsan_init_arg(size_t n)
 388 {
 389         msan_td_t *mtd;
 390         uint8_t *arg;
 391
 392         if (__predict_false(!kmsan_enabled))
 393                 return;
 394         if (__predict_false(curthread == NULL))
 395                 return;
 396         mtd = curthread->td_kmsan;
 397         arg = mtd->tls[mtd->ctx].param_shadow;
 398         __builtin_memset(arg, 0, n);
 399 }
 400
 401 void
 402 kmsan_init_ret(size_t n)
 403 {
 404         msan_td_t *mtd;
 405         uint8_t *arg;
 406
 407         if (__predict_false(!kmsan_enabled))
 408                 return;
 409         if (__predict_false(curthread == NULL))
 410                 return;
 411         mtd = curthread->td_kmsan;
 412         arg = mtd->tls[mtd->ctx].retval_shadow;
 413         __builtin_memset(arg, 0, n);
 414 }
 415
 416 static void
 417 kmsan_check_arg(size_t size, const char *hook)
 418 {
 419         msan_td_t *mtd;
 420         uint8_t *arg;
 421         size_t i;
 422
 423         if (__predict_false(!kmsan_enabled))
 424                 return;
 425         if (__predict_false(curthread == NULL))
 426                 return;
 427         mtd = curthread->td_kmsan;
 428         arg = mtd->tls[mtd->ctx].param_shadow;
 429
 430         for (i = 0; i < size; i++) {
 431                 if (__predict_true(arg[i] == 0))
 432                         continue;
 433                 kmsan_report_hook((const char *)arg + i, size, i, hook);
 434                 break;
 435         }
 436 }
 437
 438 void
 439 kmsan_thread_alloc(struct thread *td)
 440 {
 441         msan_td_t *mtd;
 442
 443         if (!kmsan_enabled)
 444                 return;
 445
 446         mtd = td->td_kmsan;
 447         if (mtd == NULL) {
 448                 /* We might be recycling a thread. */
 449                 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) +
 450                     sizeof(int));
 451                 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK);
 452         }
 453         kmsan_memset(mtd, 0, sizeof(*mtd));
 454         mtd->ctx = 0;
 455
 456         if (td->td_kstack != 0)
 457                 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages),
 458                     KMSAN_STATE_UNINIT);
 459
 460         td->td_kmsan = mtd;
 461 }
 462
 463 void
 464 kmsan_thread_free(struct thread *td)
 465 {
 466         msan_td_t *mtd;
 467
 468         if (!kmsan_enabled)
 469                 return;
 470         if (__predict_false(td == curthread))
 471                 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__);
 472
 473         mtd = td->td_kmsan;
 474         kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *));
 475         free(mtd, M_KMSAN);
 476         td->td_kmsan = NULL;
 477 }
 478
 479 void kmsan_intr_enter(void);
 480 void kmsan_intr_leave(void);
 481
 482 void
 483 kmsan_intr_enter(void)
 484 {
 485         msan_td_t *mtd;
 486
 487         if (__predict_false(!kmsan_enabled))
 488                 return;
 489
 490         mtd = curthread->td_kmsan;
 491         mtd->ctx++;
 492         if (__predict_false(mtd->ctx >= MSAN_NCONTEXT))
 493                 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
 494 }
 495
 496 void
 497 kmsan_intr_leave(void)
 498 {
 499         msan_td_t *mtd;
 500
 501         if (__predict_false(!kmsan_enabled))
 502                 return;
 503
 504         mtd = curthread->td_kmsan;
 505         if (__predict_false(mtd->ctx == 0))
 506                 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
 507         mtd->ctx--;
 508 }
 509
 510 /* -------------------------------------------------------------------------- */
 511
 512 void
 513 kmsan_shadow_map(vm_offset_t addr, size_t size)
 514 {
 515         size_t npages, i;
 516         vm_offset_t va;
 517
 518         MPASS(addr % PAGE_SIZE == 0);
 519         MPASS(size % PAGE_SIZE == 0);
 520
 521         if (!kmsan_enabled)
 522                 return;
 523
 524         npages = atop(size);
 525
 526         va = kmsan_md_addr_to_shad(addr);
 527         for (i = 0; i < npages; i++) {
 528                 pmap_san_enter(va + ptoa(i));
 529         }
 530
 531         va = kmsan_md_addr_to_orig(addr);
 532         for (i = 0; i < npages; i++) {
 533                 pmap_san_enter(va + ptoa(i));
 534         }
 535 }
 536
 537 void
 538 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc)
 539 {
 540         msan_orig_t orig;
 541
 542         orig = kmsan_md_orig_encode(type, pc);
 543         kmsan_origin_fill(addr, orig, size);
 544 }
 545
 546 void
 547 kmsan_mark(const void *addr, size_t size, uint8_t c)
 548 {
 549         kmsan_shadow_fill((uintptr_t)addr, c, size);
 550 }
 551
 552 void
 553 kmsan_mark_bio(const struct bio *bp, uint8_t c)
 554 {
 555         kmsan_mark(bp->bio_data, bp->bio_length, c);
 556 }
 557
 558 static void
 559 kmsan_mark_ccb(const union ccb *ccb, uint8_t c)
 560 {
 561         if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN)
 562                 return;
 563         if ((ccb->ccb_h.flags & CAM_DATA_MASK) != CAM_DATA_VADDR)
 564                 return;
 565
 566         switch (ccb->ccb_h.func_code) {
 567         case XPT_SCSI_IO: {
 568                 const struct ccb_scsiio *scsiio;
 569
 570                 scsiio = &ccb->ctio;
 571                 kmsan_mark(scsiio->data_ptr, scsiio->dxfer_len, c);
 572                 break;
 573         }
 574         case XPT_ATA_IO: {
 575                 const struct ccb_ataio *ataio;
 576
 577                 ataio = &ccb->ataio;
 578                 kmsan_mark(ataio->data_ptr, ataio->dxfer_len, c);
 579                 break;
 580         }
 581         case XPT_NVME_IO: {
 582                 const struct ccb_nvmeio *nvmeio;
 583
 584                 nvmeio = &ccb->nvmeio;
 585                 kmsan_mark(nvmeio->data_ptr, nvmeio->dxfer_len, c);
 586                 break;
 587         }
 588         default:
 589                 kmsan_panic("%s: unhandled CCB type %d", __func__,
 590                     ccb->ccb_h.func_code);
 591         }
 592 }
 593
 594 void
 595 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c)
 596 {
 597         do {
 598                 if ((m->m_flags & M_EXTPG) == 0)
 599                         kmsan_mark(m->m_data, m->m_len, c);
 600                 m = m->m_next;
 601         } while (m != NULL);
 602 }
 603
 604 void
 605 kmsan_check(const void *p, size_t sz, const char *descr)
 606 {
 607         kmsan_shadow_check((uintptr_t)p, sz, descr);
 608 }
 609
 610 void
 611 kmsan_check_bio(const struct bio *bp, const char *descr)
 612 {
 613         kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr);
 614 }
 615
 616 void
 617 kmsan_check_ccb(const union ccb *ccb, const char *descr)
 618 {
 619         if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT)
 620                 return;
 621         switch (ccb->ccb_h.func_code) {
 622         case XPT_SCSI_IO: {
 623                 const struct ccb_scsiio *scsiio;
 624
 625                 scsiio = &ccb->ctio;
 626                 kmsan_check(scsiio->data_ptr, scsiio->dxfer_len, descr);
 627                 break;
 628         }
 629         case XPT_ATA_IO: {
 630                 const struct ccb_ataio *ataio;
 631
 632                 ataio = &ccb->ataio;
 633                 kmsan_check(ataio->data_ptr, ataio->dxfer_len, descr);
 634                 break;
 635         }
 636         case XPT_NVME_IO: {
 637                 const struct ccb_nvmeio *nvmeio;
 638
 639                 nvmeio = &ccb->nvmeio;
 640                 kmsan_check(nvmeio->data_ptr, nvmeio->dxfer_len, descr);
 641                 break;
 642         }
 643         default:
 644                 kmsan_panic("%s: unhandled CCB type %d", __func__,
 645                     ccb->ccb_h.func_code);
 646         }
 647 }
 648
 649 void
 650 kmsan_check_mbuf(const struct mbuf *m, const char *descr)
 651 {
 652         do {
 653                 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr);
 654         } while ((m = m->m_next) != NULL);
 655 }
 656
 657 void
 658 kmsan_init(void)
 659 {
 660         int disabled;
 661
 662         disabled = 0;
 663         TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled);
 664         if (disabled)
 665                 return;
 666
 667         /* Initialize the TLS for curthread. */
 668         msan_thread0.ctx = 0;
 669         thread0.td_kmsan = &msan_thread0;
 670
 671         /* Now officially enabled. */
 672         kmsan_enabled = true;
 673 }
 674
 675 /* -------------------------------------------------------------------------- */
 676
 677 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
 678 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
 679
 680 msan_meta_t
 681 __msan_metadata_ptr_for_load_n(void *addr, size_t size)
 682 {
 683         return (kmsan_meta_get(addr, size, false));
 684 }
 685
 686 msan_meta_t
 687 __msan_metadata_ptr_for_store_n(void *addr, size_t size)
 688 {
 689         return (kmsan_meta_get(addr, size, true));
 690 }
 691
 692 #define MSAN_META_FUNC(size)                                            \
 693         msan_meta_t __msan_metadata_ptr_for_load_##size(void *);        \
 694         msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr)     \
 695         {                                                               \
 696                 return (kmsan_meta_get(addr, size, false));             \
 697         }                                                               \
 698         msan_meta_t __msan_metadata_ptr_for_store_##size(void *);       \
 699         msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr)    \
 700         {                                                               \
 701                 return (kmsan_meta_get(addr, size, true));              \
 702         }
 703
 704 MSAN_META_FUNC(1)
 705 MSAN_META_FUNC(2)
 706 MSAN_META_FUNC(4)
 707 MSAN_META_FUNC(8)
 708
 709 void __msan_instrument_asm_store(const void *, size_t);
 710 msan_orig_t __msan_chain_origin(msan_orig_t);
 711 void __msan_poison(const void *, size_t);
 712 void __msan_unpoison(const void *, size_t);
 713 void __msan_poison_alloca(const void *, uint64_t, const char *);
 714 void __msan_unpoison_alloca(const void *, uint64_t);
 715 void __msan_warning(msan_orig_t);
 716 msan_tls_t *__msan_get_context_state(void);
 717
 718 void
 719 __msan_instrument_asm_store(const void *addr, size_t size)
 720 {
 721         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
 722 }
 723
 724 msan_orig_t
 725 __msan_chain_origin(msan_orig_t origin)
 726 {
 727         return (origin);
 728 }
 729
 730 void
 731 __msan_poison(const void *addr, size_t size)
 732 {
 733         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
 734 }
 735
 736 void
 737 __msan_unpoison(const void *addr, size_t size)
 738 {
 739         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
 740 }
 741
 742 void
 743 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr)
 744 {
 745         msan_orig_t orig;
 746
 747         orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
 748         kmsan_origin_fill(addr, orig, size);
 749         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
 750 }
 751
 752 void
 753 __msan_unpoison_alloca(const void *addr, uint64_t size)
 754 {
 755         kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
 756 }
 757
 758 void
 759 __msan_warning(msan_orig_t origin)
 760 {
 761         if (__predict_false(!kmsan_enabled))
 762                 return;
 763         kmsan_report_inline(origin, KMSAN_RET_ADDR);
 764 }
 765
 766 msan_tls_t *
 767 __msan_get_context_state(void)
 768 {
 769         msan_td_t *mtd;
 770
 771         /*
 772          * When APs are started, they execute some C code before curthread is
 773          * set.  We have to handle that here.
 774          */
 775         if (__predict_false(!kmsan_enabled || curthread == NULL))
 776                 return (&dummy_tls);
 777         mtd = curthread->td_kmsan;
 778         return (&mtd->tls[mtd->ctx]);
 779 }
 780
 781 /* -------------------------------------------------------------------------- */
 782
 783 /*
 784  * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
 785  * initialized areas properly.
 786  */
 787
 788 void *
 789 kmsan_memcpy(void *dst, const void *src, size_t len)
 790 {
 791         /* No kmsan_check_arg, because inlined. */
 792         kmsan_init_ret(sizeof(void *));
 793         if (__predict_true(len != 0)) {
 794                 kmsan_meta_copy(dst, src, len);
 795         }
 796         return (__builtin_memcpy(dst, src, len));
 797 }
 798
 799 int
 800 kmsan_memcmp(const void *b1, const void *b2, size_t len)
 801 {
 802         const uint8_t *_b1 = b1, *_b2 = b2;
 803         size_t i;
 804
 805         kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len),
 806             "memcmp():args");
 807         kmsan_init_ret(sizeof(int));
 808
 809         for (i = 0; i < len; i++) {
 810                 if (*_b1 != *_b2) {
 811                         kmsan_shadow_check((uintptr_t)b1, i + 1,
 812                             "memcmp():arg1");
 813                         kmsan_shadow_check((uintptr_t)b2, i + 1,
 814                             "memcmp():arg2");
 815                         return (*_b1 - *_b2);
 816                 }
 817                 _b1++, _b2++;
 818         }
 819
 820         return (0);
 821 }
 822
 823 void *
 824 kmsan_memset(void *dst, int c, size_t len)
 825 {
 826         /* No kmsan_check_arg, because inlined. */
 827         kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len);
 828         kmsan_init_ret(sizeof(void *));
 829         return (__builtin_memset(dst, c, len));
 830 }
 831
 832 void *
 833 kmsan_memmove(void *dst, const void *src, size_t len)
 834 {
 835         /* No kmsan_check_arg, because inlined. */
 836         if (__predict_true(len != 0)) {
 837                 kmsan_meta_copy(dst, src, len);
 838         }
 839         kmsan_init_ret(sizeof(void *));
 840         return (__builtin_memmove(dst, src, len));
 841 }
 842
 843 __strong_reference(kmsan_memcpy, __msan_memcpy);
 844 __strong_reference(kmsan_memset, __msan_memset);
 845 __strong_reference(kmsan_memmove, __msan_memmove);
 846
 847 char *
 848 kmsan_strcpy(char *dst, const char *src)
 849 {
 850         const char *_src = src;
 851         char *_dst = dst;
 852         size_t len = 0;
 853
 854         kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args");
 855
 856         while (1) {
 857                 len++;
 858                 *dst = *src;
 859                 if (*src == '\0')
 860                         break;
 861                 src++, dst++;
 862         }
 863
 864         kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2");
 865         kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len);
 866         kmsan_init_ret(sizeof(char *));
 867         return (_dst);
 868 }
 869
 870 int
 871 kmsan_strcmp(const char *s1, const char *s2)
 872 {
 873         const char *_s1 = s1, *_s2 = s2;
 874         size_t len = 0;
 875
 876         kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args");
 877         kmsan_init_ret(sizeof(int));
 878
 879         while (1) {
 880                 len++;
 881                 if (*s1 != *s2)
 882                         break;
 883                 if (*s1 == '\0') {
 884                         kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
 885                         kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
 886                         return (0);
 887                 }
 888                 s1++, s2++;
 889         }
 890
 891         kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
 892         kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
 893
 894         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 895 }
 896
 897 size_t
 898 kmsan_strlen(const char *str)
 899 {
 900         const char *s;
 901
 902         kmsan_check_arg(sizeof(str), "strlen():args");
 903
 904         s = str;
 905         while (1) {
 906                 if (*s == '\0')
 907                         break;
 908                 s++;
 909         }
 910
 911         kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1");
 912         kmsan_init_ret(sizeof(size_t));
 913         return (s - str);
 914 }
 915
 916 int     kmsan_copyin(const void *, void *, size_t);
 917 int     kmsan_copyout(const void *, void *, size_t);
 918 int     kmsan_copyinstr(const void *, void *, size_t, size_t *);
 919
 920 int
 921 kmsan_copyin(const void *uaddr, void *kaddr, size_t len)
 922 {
 923         int ret;
 924
 925         kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len),
 926             "copyin():args");
 927         ret = copyin(uaddr, kaddr, len);
 928         if (ret == 0)
 929                 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len);
 930         kmsan_init_ret(sizeof(int));
 931         return (ret);
 932 }
 933
 934 int
 935 kmsan_copyout(const void *kaddr, void *uaddr, size_t len)
 936 {
 937         kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len),
 938             "copyout():args");
 939         kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1");
 940         kmsan_init_ret(sizeof(int));
 941         return (copyout(kaddr, uaddr, len));
 942 }
 943
 944 int
 945 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 946 {
 947         size_t _done;
 948         int ret;
 949
 950         kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) +
 951             sizeof(len) + sizeof(done), "copyinstr():args");
 952         ret = copyinstr(uaddr, kaddr, len, &_done);
 953         if (ret == 0)
 954                 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done);
 955         if (done != NULL) {
 956                 *done = _done;
 957                 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t));
 958         }
 959         kmsan_init_ret(sizeof(int));
 960         return (ret);
 961 }
 962
 963 /* -------------------------------------------------------------------------- */
 964
 965 int
 966 kmsan_fubyte(volatile const void *base)
 967 {
 968         int ret;
 969
 970         kmsan_check_arg(sizeof(base), "fubyte(): args");
 971         ret = fubyte(base);
 972         kmsan_init_ret(sizeof(int));
 973         return (ret);
 974 }
 975
 976 int
 977 kmsan_fuword16(volatile const void *base)
 978 {
 979         int ret;
 980
 981         kmsan_check_arg(sizeof(base), "fuword16(): args");
 982         ret = fuword16(base);
 983         kmsan_init_ret(sizeof(int));
 984         return (ret);
 985 }
 986
 987 int
 988 kmsan_fueword(volatile const void *base, long *val)
 989 {
 990         int ret;
 991
 992         kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args");
 993         ret = fueword(base, val);
 994         if (ret == 0)
 995                 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
 996                     sizeof(*val));
 997         kmsan_init_ret(sizeof(int));
 998         return (ret);
 999 }
1000
1001 int
1002 kmsan_fueword32(volatile const void *base, int32_t *val)
1003 {
1004         int ret;
1005
1006         kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args");
1007         ret = fueword32(base, val);
1008         if (ret == 0)
1009                 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
1010                     sizeof(*val));
1011         kmsan_init_ret(sizeof(int));
1012         return (ret);
1013 }
1014
1015 int
1016 kmsan_fueword64(volatile const void *base, int64_t *val)
1017 {
1018         int ret;
1019
1020         kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args");
1021         ret = fueword64(base, val);
1022         if (ret == 0)
1023                 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
1024                     sizeof(*val));
1025         kmsan_init_ret(sizeof(int));
1026         return (ret);
1027 }
1028
1029 int
1030 kmsan_subyte(volatile void *base, int byte)
1031 {
1032         int ret;
1033
1034         kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args");
1035         ret = subyte(base, byte);
1036         kmsan_init_ret(sizeof(int));
1037         return (ret);
1038 }
1039
1040 int
1041 kmsan_suword(volatile void *base, long word)
1042 {
1043         int ret;
1044
1045         kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args");
1046         ret = suword(base, word);
1047         kmsan_init_ret(sizeof(int));
1048         return (ret);
1049 }
1050
1051 int
1052 kmsan_suword16(volatile void *base, int word)
1053 {
1054         int ret;
1055
1056         kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args");
1057         ret = suword16(base, word);
1058         kmsan_init_ret(sizeof(int));
1059         return (ret);
1060 }
1061
1062 int
1063 kmsan_suword32(volatile void *base, int32_t word)
1064 {
1065         int ret;
1066
1067         kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args");
1068         ret = suword32(base, word);
1069         kmsan_init_ret(sizeof(int));
1070         return (ret);
1071 }
1072
1073 int
1074 kmsan_suword64(volatile void *base, int64_t word)
1075 {
1076         int ret;
1077
1078         kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args");
1079         ret = suword64(base, word);
1080         kmsan_init_ret(sizeof(int));
1081         return (ret);
1082 }
1083
1084 int
1085 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
1086     uint32_t newval)
1087 {
1088         int ret;
1089
1090         kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1091             sizeof(newval), "casueword32(): args");
1092         ret = casueword32(base, oldval, oldvalp, newval);
1093         kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1094             sizeof(*oldvalp));
1095         kmsan_init_ret(sizeof(int));
1096         return (ret);
1097 }
1098
1099 int
1100 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
1101     u_long newval)
1102 {
1103         int ret;
1104
1105         kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1106             sizeof(newval), "casueword32(): args");
1107         ret = casueword(base, oldval, oldvalp, newval);
1108         kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1109             sizeof(*oldvalp));
1110         kmsan_init_ret(sizeof(int));
1111         return (ret);
1112 }
1113
1114 /* -------------------------------------------------------------------------- */
1115
1116 #include <machine/atomic.h>
1117 #include <sys/atomic_san.h>
1118
1119 #define _MSAN_ATOMIC_FUNC_ADD(name, type)                               \
1120         void kmsan_atomic_add_##name(volatile type *ptr, type val)      \
1121         {                                                               \
1122                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1123                     "atomic_add_" #name "():args");                     \
1124                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1125                     "atomic_add_" #name "():ptr");                      \
1126                 atomic_add_##name(ptr, val);                            \
1127         }
1128
1129 #define MSAN_ATOMIC_FUNC_ADD(name, type)                                \
1130         _MSAN_ATOMIC_FUNC_ADD(name, type)                               \
1131         _MSAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
1132         _MSAN_ATOMIC_FUNC_ADD(rel_##name, type)
1133
1134 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
1135         void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \
1136         {                                                               \
1137                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1138                     "atomic_subtract_" #name "():args");                \
1139                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1140                     "atomic_subtract_" #name "():ptr");                 \
1141                 atomic_subtract_##name(ptr, val);                       \
1142         }
1143
1144 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
1145         _MSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
1146         _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
1147         _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
1148
1149 #define _MSAN_ATOMIC_FUNC_SET(name, type)                               \
1150         void kmsan_atomic_set_##name(volatile type *ptr, type val)      \
1151         {                                                               \
1152                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1153                     "atomic_set_" #name "():args");                     \
1154                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1155                     "atomic_set_" #name "():ptr");                      \
1156                 atomic_set_##name(ptr, val);                            \
1157         }
1158
1159 #define MSAN_ATOMIC_FUNC_SET(name, type)                                \
1160         _MSAN_ATOMIC_FUNC_SET(name, type)                               \
1161         _MSAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
1162         _MSAN_ATOMIC_FUNC_SET(rel_##name, type)
1163
1164 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
1165         void kmsan_atomic_clear_##name(volatile type *ptr, type val)    \
1166         {                                                               \
1167                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1168                     "atomic_clear_" #name "():args");                   \
1169                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1170                     "atomic_clear_" #name "():ptr");                    \
1171                 atomic_clear_##name(ptr, val);                          \
1172         }
1173
1174 #define MSAN_ATOMIC_FUNC_CLEAR(name, type)                              \
1175         _MSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
1176         _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
1177         _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
1178
1179 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
1180         type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
1181         {                                                               \
1182                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1183                     "atomic_fetchadd_" #name "():args");                \
1184                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1185                     "atomic_fetchadd_" #name "():ptr");                 \
1186                 kmsan_init_ret(sizeof(type));                           \
1187                 return (atomic_fetchadd_##name(ptr, val));              \
1188         }
1189
1190 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
1191         type kmsan_atomic_readandclear_##name(volatile type *ptr)       \
1192         {                                                               \
1193                 kmsan_check_arg(sizeof(ptr),                            \
1194                     "atomic_readandclear_" #name "():args");            \
1195                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1196                     "atomic_readandclear_" #name "():ptr");             \
1197                 kmsan_init_ret(sizeof(type));                           \
1198                 return (atomic_readandclear_##name(ptr));               \
1199         }
1200
1201 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
1202         int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
1203         {                                                               \
1204                 kmsan_check_arg(sizeof(ptr) + sizeof(v),                \
1205                     "atomic_testandclear_" #name "():args");            \
1206                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1207                     "atomic_testandclear_" #name "():ptr");             \
1208                 kmsan_init_ret(sizeof(int));                            \
1209                 return (atomic_testandclear_##name(ptr, v));            \
1210         }
1211
1212 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
1213         int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \
1214         {                                                               \
1215                 kmsan_check_arg(sizeof(ptr) + sizeof(v),                \
1216                     "atomic_testandset_" #name "():args");              \
1217                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1218                     "atomic_testandset_" #name "():ptr");               \
1219                 kmsan_init_ret(sizeof(int));                            \
1220                 return (atomic_testandset_##name(ptr, v));              \
1221         }
1222
1223 #define MSAN_ATOMIC_FUNC_SWAP(name, type)                               \
1224         type kmsan_atomic_swap_##name(volatile type *ptr, type val)     \
1225         {                                                               \
1226                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1227                     "atomic_swap_" #name "():args");                    \
1228                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1229                     "atomic_swap_" #name "():ptr");                     \
1230                 kmsan_init_ret(sizeof(type));                           \
1231                 return (atomic_swap_##name(ptr, val));                  \
1232         }
1233
1234 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
1235         int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval,   \
1236             type nval)                                                  \
1237         {                                                               \
1238                 kmsan_check_arg(sizeof(ptr) + sizeof(oval) +            \
1239                     sizeof(nval), "atomic_cmpset_" #name "():args");    \
1240                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1241                     "atomic_cmpset_" #name "():ptr");                   \
1242                 kmsan_init_ret(sizeof(int));                            \
1243                 return (atomic_cmpset_##name(ptr, oval, nval));         \
1244         }
1245
1246 #define MSAN_ATOMIC_FUNC_CMPSET(name, type)                             \
1247         _MSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
1248         _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
1249         _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
1250
1251 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
1252         int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
1253             type nval)                                                  \
1254         {                                                               \
1255                 kmsan_check_arg(sizeof(ptr) + sizeof(oval) +            \
1256                     sizeof(nval), "atomic_fcmpset_" #name "():args");   \
1257                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1258                     "atomic_fcmpset_" #name "():ptr");                  \
1259                 kmsan_init_ret(sizeof(int));                            \
1260                 return (atomic_fcmpset_##name(ptr, oval, nval));        \
1261         }
1262
1263 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
1264         _MSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
1265         _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
1266         _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
1267
1268 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
1269         void kmsan_atomic_thread_fence_##name(void)                     \
1270         {                                                               \
1271                 atomic_thread_fence_##name();                           \
1272         }
1273
1274 #define _MSAN_ATOMIC_FUNC_LOAD(name, type)                              \
1275         type kmsan_atomic_load_##name(volatile type *ptr)               \
1276         {                                                               \
1277                 kmsan_check_arg(sizeof(ptr),                            \
1278                     "atomic_load_" #name "():args");                    \
1279                 kmsan_shadow_check((uintptr_t)ptr, sizeof(type),        \
1280                     "atomic_load_" #name "():ptr");                     \
1281                 kmsan_init_ret(sizeof(type));                           \
1282                 return (atomic_load_##name(ptr));                       \
1283         }
1284
1285 #define MSAN_ATOMIC_FUNC_LOAD(name, type)                               \
1286         _MSAN_ATOMIC_FUNC_LOAD(name, type)                              \
1287         _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type)
1288
1289 #define _MSAN_ATOMIC_FUNC_STORE(name, type)                             \
1290         void kmsan_atomic_store_##name(volatile type *ptr, type val)    \
1291         {                                                               \
1292                 kmsan_check_arg(sizeof(ptr) + sizeof(val),              \
1293                     "atomic_store_" #name "():args");                   \
1294                 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED,   \
1295                     sizeof(type));                                      \
1296                 atomic_store_##name(ptr, val);                          \
1297         }
1298
1299 #define MSAN_ATOMIC_FUNC_STORE(name, type)                              \
1300         _MSAN_ATOMIC_FUNC_STORE(name, type)                             \
1301         _MSAN_ATOMIC_FUNC_STORE(rel_##name, type)
1302
1303 MSAN_ATOMIC_FUNC_ADD(8, uint8_t);
1304 MSAN_ATOMIC_FUNC_ADD(16, uint16_t);
1305 MSAN_ATOMIC_FUNC_ADD(32, uint32_t);
1306 MSAN_ATOMIC_FUNC_ADD(64, uint64_t);
1307 MSAN_ATOMIC_FUNC_ADD(int, u_int);
1308 MSAN_ATOMIC_FUNC_ADD(long, u_long);
1309 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
1310
1311 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
1312 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
1313 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
1314 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
1315 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
1316 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
1317 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
1318
1319 MSAN_ATOMIC_FUNC_SET(8, uint8_t);
1320 MSAN_ATOMIC_FUNC_SET(16, uint16_t);
1321 MSAN_ATOMIC_FUNC_SET(32, uint32_t);
1322 MSAN_ATOMIC_FUNC_SET(64, uint64_t);
1323 MSAN_ATOMIC_FUNC_SET(int, u_int);
1324 MSAN_ATOMIC_FUNC_SET(long, u_long);
1325 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
1326
1327 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
1328 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
1329 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
1330 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
1331 MSAN_ATOMIC_FUNC_CLEAR(int, u_int);
1332 MSAN_ATOMIC_FUNC_CLEAR(long, u_long);
1333 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
1334
1335 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
1336 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
1337 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int);
1338 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long);
1339
1340 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
1341 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
1342 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
1343 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
1344 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
1345
1346 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
1347 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
1348 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
1349 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
1350
1351 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
1352 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
1353 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
1354 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
1355
1356 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t);
1357 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t);
1358 MSAN_ATOMIC_FUNC_SWAP(int, u_int);
1359 MSAN_ATOMIC_FUNC_SWAP(long, u_long);
1360 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
1361
1362 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
1363 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
1364 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
1365 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
1366 MSAN_ATOMIC_FUNC_CMPSET(int, u_int);
1367 MSAN_ATOMIC_FUNC_CMPSET(long, u_long);
1368 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
1369
1370 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
1371 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
1372 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
1373 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
1374 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int);
1375 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long);
1376 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
1377
1378 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t);
1379 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t);
1380 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t);
1381 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t);
1382 MSAN_ATOMIC_FUNC_LOAD(char, u_char);
1383 MSAN_ATOMIC_FUNC_LOAD(short, u_short);
1384 MSAN_ATOMIC_FUNC_LOAD(int, u_int);
1385 MSAN_ATOMIC_FUNC_LOAD(long, u_long);
1386 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
1387
1388 MSAN_ATOMIC_FUNC_STORE(8, uint8_t);
1389 MSAN_ATOMIC_FUNC_STORE(16, uint16_t);
1390 MSAN_ATOMIC_FUNC_STORE(32, uint32_t);
1391 MSAN_ATOMIC_FUNC_STORE(64, uint64_t);
1392 MSAN_ATOMIC_FUNC_STORE(char, u_char);
1393 MSAN_ATOMIC_FUNC_STORE(short, u_short);
1394 MSAN_ATOMIC_FUNC_STORE(int, u_int);
1395 MSAN_ATOMIC_FUNC_STORE(long, u_long);
1396 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
1397
1398 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq);
1399 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel);
1400 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
1401 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
1402
1403 void
1404 kmsan_atomic_interrupt_fence(void)
1405 {
1406         atomic_interrupt_fence();
1407 }
1408
1409 /* -------------------------------------------------------------------------- */
1410
1411 #include <sys/bus.h>
1412 #include <machine/bus.h>
1413 #include <sys/bus_san.h>
1414
1415 int
1416 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
1417     int flags, bus_space_handle_t *handlep)
1418 {
1419         return (bus_space_map(tag, hnd, size, flags, handlep));
1420 }
1421
1422 void
1423 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
1424     bus_size_t size)
1425 {
1426         bus_space_unmap(tag, hnd, size);
1427 }
1428
1429 int
1430 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
1431     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
1432 {
1433         return (bus_space_subregion(tag, hnd, offset, size, handlep));
1434 }
1435
1436 void
1437 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
1438     bus_size_t size)
1439 {
1440         bus_space_free(tag, hnd, size);
1441 }
1442
1443 void
1444 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
1445     bus_size_t offset, bus_size_t size, int flags)
1446 {
1447         bus_space_barrier(tag, hnd, offset, size, flags);
1448 }
1449
1450 /* XXXMJ x86-specific */
1451 #define MSAN_BUS_READ_FUNC(func, width, type)                           \
1452         type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
1453             bus_space_handle_t hnd, bus_size_t offset)                  \
1454         {                                                               \
1455                 type ret;                                               \
1456                 if ((tag) != X86_BUS_SPACE_IO)                          \
1457                         kmsan_shadow_fill((uintptr_t)(hnd + offset),    \
1458                             KMSAN_STATE_INITED, (width));               \
1459                 ret = bus_space_read##func##_##width(tag, hnd, offset); \
1460                 kmsan_init_ret(sizeof(type));                           \
1461                 return (ret);                                           \
1462         }                                                               \
1463
1464 #define MSAN_BUS_READ_PTR_FUNC(func, width, type)                       \
1465         void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
1466             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
1467             bus_size_t count)                                           \
1468         {                                                               \
1469                 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED,   \
1470                     (width) * count);                                   \
1471                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
1472                     count);                                             \
1473         }
1474
1475 MSAN_BUS_READ_FUNC(, 1, uint8_t)
1476 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
1477 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
1478 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
1479 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
1480 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
1481
1482 MSAN_BUS_READ_FUNC(, 2, uint16_t)
1483 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
1484 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
1485 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
1486 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
1487 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
1488
1489 MSAN_BUS_READ_FUNC(, 4, uint32_t)
1490 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
1491 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
1492 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
1493 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
1494 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
1495
1496 MSAN_BUS_READ_FUNC(, 8, uint64_t)
1497
1498 #define MSAN_BUS_WRITE_FUNC(func, width, type)                          \
1499         void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
1500             bus_space_handle_t hnd, bus_size_t offset, type value)      \
1501         {                                                               \
1502                 bus_space_write##func##_##width(tag, hnd, offset, value);\
1503         }                                                               \
1504
1505 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
1506         void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
1507             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
1508             bus_size_t count)                                           \
1509         {                                                               \
1510                 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
1511                     "bus_space_write()");                               \
1512                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
1513                     count);                                             \
1514         }
1515
1516 MSAN_BUS_WRITE_FUNC(, 1, uint8_t)
1517 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
1518 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
1519 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
1520 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
1521 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
1522
1523 MSAN_BUS_WRITE_FUNC(, 2, uint16_t)
1524 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
1525 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
1526 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
1527 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1528 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1529
1530 MSAN_BUS_WRITE_FUNC(, 4, uint32_t)
1531 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1532 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1533 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1534 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1535 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1536
1537 MSAN_BUS_WRITE_FUNC(, 8, uint64_t)
1538
1539 #define MSAN_BUS_SET_FUNC(func, width, type)                            \
1540         void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
1541             bus_space_handle_t hnd, bus_size_t offset, type value,      \
1542             bus_size_t count)                                           \
1543         {                                                               \
1544                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1545                     count);                                             \
1546         }
1547
1548 MSAN_BUS_SET_FUNC(multi, 1, uint8_t)
1549 MSAN_BUS_SET_FUNC(region, 1, uint8_t)
1550 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1551 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1552
1553 MSAN_BUS_SET_FUNC(multi, 2, uint16_t)
1554 MSAN_BUS_SET_FUNC(region, 2, uint16_t)
1555 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1556 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1557
1558 MSAN_BUS_SET_FUNC(multi, 4, uint32_t)
1559 MSAN_BUS_SET_FUNC(region, 4, uint32_t)
1560 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1561 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1562
1563 /* -------------------------------------------------------------------------- */
1564
1565 void
1566 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op)
1567 {
1568         /*
1569          * Some drivers, e.g., nvme, use the same code path for loading device
1570          * read and write requests, and will thus specify both flags.  In this
1571          * case we should not do any checking since it will generally lead to
1572          * false positives.
1573          */
1574         if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) ==
1575             BUS_DMASYNC_PREWRITE) {
1576                 switch (desc->md_type) {
1577                 case MEMDESC_VADDR:
1578                         kmsan_check(desc->u.md_vaddr, desc->md_opaque,
1579                             "dmasync");
1580                         break;
1581                 case MEMDESC_BIO:
1582                         kmsan_check_bio(desc->u.md_bio, "dmasync");
1583                         break;
1584                 case MEMDESC_MBUF:
1585                         kmsan_check_mbuf(desc->u.md_mbuf, "dmasync");
1586                         break;
1587                 case MEMDESC_CCB:
1588                         kmsan_check_ccb(desc->u.md_ccb, "dmasync");
1589                         break;
1590                 case 0:
1591                         break;
1592                 default:
1593                         kmsan_panic("%s: unhandled memdesc type %d", __func__,
1594                             desc->md_type);
1595                 }
1596         }
1597         if ((op & BUS_DMASYNC_POSTREAD) != 0) {
1598                 switch (desc->md_type) {
1599                 case MEMDESC_VADDR:
1600                         kmsan_mark(desc->u.md_vaddr, desc->md_opaque,
1601                             KMSAN_STATE_INITED);
1602                         break;
1603                 case MEMDESC_BIO:
1604                         kmsan_mark_bio(desc->u.md_bio, KMSAN_STATE_INITED);
1605                         break;
1606                 case MEMDESC_MBUF:
1607                         kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED);
1608                         break;
1609                 case MEMDESC_CCB:
1610                         kmsan_mark_ccb(desc->u.md_ccb, KMSAN_STATE_INITED);
1611                         break;
1612                 case 0:
1613                         break;
1614                 default:
1615                         kmsan_panic("%s: unhandled memdesc type %d", __func__,
1616                             desc->md_type);
1617                 }
1618         }
1619 }