sys/kern/subr_csan.c

   1 /*      $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $       */
   2
   3 /*
   4  * Copyright (c) 2019 The NetBSD Foundation, Inc.
   5  * All rights reserved.
   6  * Copyright (c) 2019 Andrew Turner
   7  *
   8  * This code is derived from software contributed to The NetBSD Foundation
   9  * by Maxime Villard.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30  * POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #define SAN_RUNTIME
  34
  35 #include "opt_ddb.h"
  36
  37 #include <sys/param.h>
  38 #include <sys/kernel.h>
  39 #include <sys/bus.h>
  40 #include <sys/conf.h>
  41 #include <sys/cpu.h>
  42 #include <sys/csan.h>
  43 #include <sys/proc.h>
  44 #include <sys/smp.h>
  45 #include <sys/systm.h>
  46
  47 #include <ddb/ddb.h>
  48 #include <ddb/db_sym.h>
  49
  50 #ifdef KCSAN_PANIC
  51 #define REPORT panic
  52 #else
  53 #define REPORT printf
  54 #endif
  55
  56 typedef struct {
  57         uintptr_t addr;
  58         uint32_t size;
  59         bool write:1;
  60         bool atomic:1;
  61         uintptr_t pc;
  62 } csan_cell_t;
  63
  64 typedef struct {
  65         bool inited;
  66         uint32_t cnt;
  67         csan_cell_t cell;
  68 } csan_cpu_t;
  69
  70 static csan_cpu_t kcsan_cpus[MAXCPU];
  71 static bool kcsan_enabled __read_mostly;
  72
  73 #define __RET_ADDR      (uintptr_t)__builtin_return_address(0)
  74
  75 #define KCSAN_NACCESSES 1024
  76 #define KCSAN_DELAY     10      /* 10 microseconds */
  77
  78 /* -------------------------------------------------------------------------- */
  79
  80 /* The MD code. */
  81 #include <machine/csan.h>
  82
  83 /* -------------------------------------------------------------------------- */
  84
  85 static void
  86 kcsan_enable(void *dummy __unused)
  87 {
  88
  89         printf("Enabling KCSCAN, expect reduced performance.\n");
  90         kcsan_enabled = true;
  91 }
  92 SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL);
  93
  94 void
  95 kcsan_cpu_init(u_int cpu)
  96 {
  97         kcsan_cpus[cpu].inited = true;
  98 }
  99
 100 /* -------------------------------------------------------------------------- */
 101
 102 static inline void
 103 kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu)
 104 {
 105         const char *newsym, *oldsym;
 106 #ifdef DDB
 107         c_db_sym_t sym;
 108         db_expr_t offset;
 109
 110         sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset);
 111         db_symbol_values(sym, &newsym, NULL);
 112
 113         sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset);
 114         db_symbol_values(sym, &oldsym, NULL);
 115 #else
 116         newsym = "";
 117         oldsym = "";
 118 #endif
 119         REPORT("CSan: Racy Access "
 120             "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] "
 121             "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n",
 122             newcpu,
 123             (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"),
 124             (void *)new->addr, new->size, (void *)new->pc, newsym,
 125             oldcpu,
 126             (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"),
 127             (void *)old->addr, old->size, (void *)old->pc, oldsym);
 128         kcsan_md_unwind();
 129 }
 130
 131 static inline bool
 132 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old)
 133 {
 134         if (new->write && !new->atomic)
 135                 return false;
 136         if (old->write && !old->atomic)
 137                 return false;
 138         return true;
 139 }
 140
 141 static inline void
 142 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc)
 143 {
 144         csan_cell_t old, new;
 145         csan_cpu_t *cpu;
 146         uint64_t intr;
 147         size_t i;
 148
 149         if (__predict_false(!kcsan_enabled))
 150                 return;
 151         if (__predict_false(kcsan_md_unsupported((vm_offset_t)addr)))
 152                 return;
 153         if (KERNEL_PANICKED())
 154                 return;
 155
 156         new.addr = addr;
 157         new.size = size;
 158         new.write = write;
 159         new.atomic = atomic;
 160         new.pc = pc;
 161
 162         CPU_FOREACH(i) {
 163                 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old));
 164
 165                 if (old.addr + old.size <= new.addr)
 166                         continue;
 167                 if (new.addr + new.size <= old.addr)
 168                         continue;
 169                 if (__predict_true(!old.write && !new.write))
 170                         continue;
 171                 if (__predict_true(kcsan_access_is_atomic(&new, &old)))
 172                         continue;
 173
 174                 kcsan_report(&new, PCPU_GET(cpuid), &old, i);
 175                 break;
 176         }
 177
 178         if (__predict_false(!kcsan_md_is_avail()))
 179                 return;
 180
 181         kcsan_md_disable_intrs(&intr);
 182
 183         cpu = &kcsan_cpus[PCPU_GET(cpuid)];
 184         if (__predict_false(!cpu->inited))
 185                 goto out;
 186         cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES;
 187         if (__predict_true(cpu->cnt != 0))
 188                 goto out;
 189
 190         __builtin_memcpy(&cpu->cell, &new, sizeof(new));
 191         kcsan_md_delay(KCSAN_DELAY);
 192         __builtin_memset(&cpu->cell, 0, sizeof(new));
 193
 194 out:
 195         kcsan_md_enable_intrs(&intr);
 196 }
 197
 198 #define CSAN_READ(size)                                                 \
 199         void __tsan_read##size(uintptr_t);                              \
 200         void __tsan_read##size(uintptr_t addr)                          \
 201         {                                                               \
 202                 kcsan_access(addr, size, false, false, __RET_ADDR);     \
 203         }                                                               \
 204         void __tsan_unaligned_read##size(uintptr_t);                    \
 205         void __tsan_unaligned_read##size(uintptr_t addr)                \
 206         {                                                               \
 207                 kcsan_access(addr, size, false, false, __RET_ADDR);     \
 208         }
 209
 210 CSAN_READ(1)
 211 CSAN_READ(2)
 212 CSAN_READ(4)
 213 CSAN_READ(8)
 214 CSAN_READ(16)
 215
 216 #define CSAN_WRITE(size)                                                \
 217         void __tsan_write##size(uintptr_t);                             \
 218         void __tsan_write##size(uintptr_t addr)                         \
 219         {                                                               \
 220                 kcsan_access(addr, size, true, false, __RET_ADDR);      \
 221         }                                                               \
 222         void __tsan_unaligned_write##size(uintptr_t);                   \
 223         void __tsan_unaligned_write##size(uintptr_t addr)               \
 224         {                                                               \
 225                 kcsan_access(addr, size, true, false, __RET_ADDR);      \
 226         }
 227
 228 CSAN_WRITE(1)
 229 CSAN_WRITE(2)
 230 CSAN_WRITE(4)
 231 CSAN_WRITE(8)
 232 CSAN_WRITE(16)
 233
 234 void __tsan_read_range(uintptr_t, size_t);
 235 void __tsan_write_range(uintptr_t, size_t);
 236
 237 void
 238 __tsan_read_range(uintptr_t addr, size_t size)
 239 {
 240         kcsan_access(addr, size, false, false, __RET_ADDR);
 241 }
 242
 243 void
 244 __tsan_write_range(uintptr_t addr, size_t size)
 245 {
 246         kcsan_access(addr, size, true, false, __RET_ADDR);
 247 }
 248
 249 void __tsan_init(void);
 250 void __tsan_func_entry(void *);
 251 void __tsan_func_exit(void);
 252
 253 void
 254 __tsan_init(void)
 255 {
 256 }
 257
 258 void
 259 __tsan_func_entry(void *call_pc)
 260 {
 261 }
 262
 263 void
 264 __tsan_func_exit(void)
 265 {
 266 }
 267
 268 /* -------------------------------------------------------------------------- */
 269
 270 void *
 271 kcsan_memcpy(void *dst, const void *src, size_t len)
 272 {
 273         kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
 274         kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
 275         return __builtin_memcpy(dst, src, len);
 276 }
 277
 278 int
 279 kcsan_memcmp(const void *b1, const void *b2, size_t len)
 280 {
 281         kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR);
 282         kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR);
 283         return __builtin_memcmp(b1, b2, len);
 284 }
 285
 286 void *
 287 kcsan_memset(void *b, int c, size_t len)
 288 {
 289         kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR);
 290         return __builtin_memset(b, c, len);
 291 }
 292
 293 void *
 294 kcsan_memmove(void *dst, const void *src, size_t len)
 295 {
 296         kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
 297         kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
 298         return __builtin_memmove(dst, src, len);
 299 }
 300
 301 __strong_reference(kcsan_memcpy, __tsan_memcpy);
 302 __strong_reference(kcsan_memset, __tsan_memset);
 303 __strong_reference(kcsan_memmove, __tsan_memmove);
 304
 305 char *
 306 kcsan_strcpy(char *dst, const char *src)
 307 {
 308         char *save = dst;
 309
 310         while (1) {
 311                 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR);
 312                 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR);
 313                 *dst = *src;
 314                 if (*src == '\0')
 315                         break;
 316                 src++, dst++;
 317         }
 318
 319         return save;
 320 }
 321
 322 int
 323 kcsan_strcmp(const char *s1, const char *s2)
 324 {
 325         while (1) {
 326                 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR);
 327                 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR);
 328                 if (*s1 != *s2)
 329                         break;
 330                 if (*s1 == '\0')
 331                         return 0;
 332                 s1++, s2++;
 333         }
 334
 335         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 336 }
 337
 338 size_t
 339 kcsan_strlen(const char *str)
 340 {
 341         const char *s;
 342
 343         s = str;
 344         while (1) {
 345                 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR);
 346                 if (*s == '\0')
 347                         break;
 348                 s++;
 349         }
 350
 351         return (s - str);
 352 }
 353
 354 int
 355 kcsan_copyin(const void *uaddr, void *kaddr, size_t len)
 356 {
 357         kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
 358         return copyin(uaddr, kaddr, len);
 359 }
 360
 361 int
 362 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 363 {
 364         kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
 365         return copyinstr(uaddr, kaddr, len, done);
 366 }
 367
 368 int
 369 kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
 370 {
 371         kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
 372         return copyout(kaddr, uaddr, len);
 373 }
 374
 375 /* -------------------------------------------------------------------------- */
 376
 377 #include <machine/atomic.h>
 378 #include <sys/atomic_san.h>
 379
 380 #define _CSAN_ATOMIC_FUNC_ADD(name, type)                               \
 381         void kcsan_atomic_add_##name(volatile type *ptr, type val)      \
 382         {                                                               \
 383                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 384                     __RET_ADDR);                                        \
 385                 atomic_add_##name(ptr, val);                            \
 386         }
 387
 388 #define CSAN_ATOMIC_FUNC_ADD(name, type)                                \
 389         _CSAN_ATOMIC_FUNC_ADD(name, type)                               \
 390         _CSAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
 391         _CSAN_ATOMIC_FUNC_ADD(rel_##name, type)
 392
 393 #define _CSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 394         void kcsan_atomic_clear_##name(volatile type *ptr, type val)    \
 395         {                                                               \
 396                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 397                     __RET_ADDR);                                        \
 398                 atomic_clear_##name(ptr, val);                          \
 399         }
 400
 401 #define CSAN_ATOMIC_FUNC_CLEAR(name, type)                              \
 402         _CSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 403         _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
 404         _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
 405
 406 #define _CSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 407         int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1,   \
 408             type val2)                                                  \
 409         {                                                               \
 410                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 411                     __RET_ADDR);                                        \
 412                 return (atomic_cmpset_##name(ptr, val1, val2));         \
 413         }
 414
 415 #define CSAN_ATOMIC_FUNC_CMPSET(name, type)                             \
 416         _CSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 417         _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
 418         _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
 419
 420 #define _CSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 421         int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \
 422             type val2)                                                  \
 423         {                                                               \
 424                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 425                     __RET_ADDR);                                        \
 426                 return (atomic_fcmpset_##name(ptr, val1, val2));        \
 427         }
 428
 429 #define CSAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
 430         _CSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 431         _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
 432         _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
 433
 434 #define CSAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
 435         type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
 436         {                                                               \
 437                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 438                     __RET_ADDR);                                        \
 439                 return (atomic_fetchadd_##name(ptr, val));              \
 440         }
 441
 442 #define _CSAN_ATOMIC_FUNC_LOAD(name, type)                              \
 443         type kcsan_atomic_load_##name(volatile type *ptr)               \
 444         {                                                               \
 445                 kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \
 446                     __RET_ADDR);                                        \
 447                 return (atomic_load_##name(ptr));                       \
 448         }
 449
 450 #define CSAN_ATOMIC_FUNC_LOAD(name, type)                               \
 451         _CSAN_ATOMIC_FUNC_LOAD(name, type)                              \
 452         _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type)                        \
 453
 454 #define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
 455         type kcsan_atomic_readandclear_##name(volatile type *ptr)       \
 456         {                                                               \
 457                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 458                     __RET_ADDR);                                        \
 459                 return (atomic_readandclear_##name(ptr));               \
 460         }
 461
 462 #define _CSAN_ATOMIC_FUNC_SET(name, type)                               \
 463         void kcsan_atomic_set_##name(volatile type *ptr, type val)      \
 464         {                                                               \
 465                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 466                     __RET_ADDR);                                        \
 467                 atomic_set_##name(ptr, val);                            \
 468         }
 469
 470 #define CSAN_ATOMIC_FUNC_SET(name, type)                                \
 471         _CSAN_ATOMIC_FUNC_SET(name, type)                               \
 472         _CSAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
 473         _CSAN_ATOMIC_FUNC_SET(rel_##name, type)
 474
 475 #define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 476         void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \
 477         {                                                               \
 478                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 479                     __RET_ADDR);                                        \
 480                 atomic_subtract_##name(ptr, val);                       \
 481         }
 482
 483 #define CSAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
 484         _CSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 485         _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
 486         _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
 487
 488 #define _CSAN_ATOMIC_FUNC_STORE(name, type)                             \
 489         void kcsan_atomic_store_##name(volatile type *ptr, type val)    \
 490         {                                                               \
 491                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 492                     __RET_ADDR);                                        \
 493                 atomic_store_##name(ptr, val);                          \
 494         }
 495
 496 #define CSAN_ATOMIC_FUNC_STORE(name, type)                              \
 497         _CSAN_ATOMIC_FUNC_STORE(name, type)                             \
 498         _CSAN_ATOMIC_FUNC_STORE(rel_##name, type)
 499
 500 #define CSAN_ATOMIC_FUNC_SWAP(name, type)                               \
 501         type kcsan_atomic_swap_##name(volatile type *ptr, type val)     \
 502         {                                                               \
 503                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 504                     __RET_ADDR);                                        \
 505                 return(atomic_swap_##name(ptr, val));                   \
 506         }
 507
 508 #define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
 509         int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \
 510         {                                                               \
 511                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 512                     __RET_ADDR);                                        \
 513                 return(atomic_testandclear_##name(ptr, val));           \
 514         }
 515
 516 #define CSAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
 517         int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \
 518         {                                                               \
 519                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 520                     __RET_ADDR);                                        \
 521                 return (atomic_testandset_##name(ptr, val));            \
 522         }
 523
 524 _CSAN_ATOMIC_FUNC_LOAD(bool, bool)
 525 _CSAN_ATOMIC_FUNC_STORE(bool, bool)
 526
 527 CSAN_ATOMIC_FUNC_ADD(8, uint8_t)
 528 CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t)
 529 CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t)
 530 CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t)
 531 CSAN_ATOMIC_FUNC_LOAD(8, uint8_t)
 532 CSAN_ATOMIC_FUNC_SET(8, uint8_t)
 533 CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t)
 534 _CSAN_ATOMIC_FUNC_STORE(8, uint8_t)
 535 #if 0
 536 CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t)
 537 CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t)
 538 CSAN_ATOMIC_FUNC_SWAP(8, uint8_t)
 539 CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t)
 540 CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t)
 541 #endif
 542
 543 CSAN_ATOMIC_FUNC_ADD(16, uint16_t)
 544 CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t)
 545 CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t)
 546 CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t)
 547 CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
 548 CSAN_ATOMIC_FUNC_SET(16, uint16_t)
 549 CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t)
 550 _CSAN_ATOMIC_FUNC_STORE(16, uint16_t)
 551 #if 0
 552 CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t)
 553 CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t)
 554 CSAN_ATOMIC_FUNC_SWAP(16, uint16_t)
 555 CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t)
 556 CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t)
 557 #endif
 558
 559 CSAN_ATOMIC_FUNC_ADD(32, uint32_t)
 560 CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t)
 561 CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t)
 562 CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t)
 563 CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t)
 564 CSAN_ATOMIC_FUNC_LOAD(32, uint32_t)
 565 CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t)
 566 CSAN_ATOMIC_FUNC_SET(32, uint32_t)
 567 CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t)
 568 CSAN_ATOMIC_FUNC_STORE(32, uint32_t)
 569 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t)
 570 CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t)
 571 CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t)
 572
 573 CSAN_ATOMIC_FUNC_ADD(64, uint64_t)
 574 CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t)
 575 CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t)
 576 CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t)
 577 CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t)
 578 CSAN_ATOMIC_FUNC_LOAD(64, uint64_t)
 579 CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t)
 580 CSAN_ATOMIC_FUNC_SET(64, uint64_t)
 581 CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t)
 582 CSAN_ATOMIC_FUNC_STORE(64, uint64_t)
 583 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t)
 584 CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t)
 585 CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t)
 586
 587 CSAN_ATOMIC_FUNC_ADD(char, uint8_t)
 588 CSAN_ATOMIC_FUNC_CLEAR(char, uint8_t)
 589 CSAN_ATOMIC_FUNC_CMPSET(char, uint8_t)
 590 CSAN_ATOMIC_FUNC_FCMPSET(char, uint8_t)
 591 CSAN_ATOMIC_FUNC_LOAD(char, uint8_t)
 592 CSAN_ATOMIC_FUNC_SET(char, uint8_t)
 593 CSAN_ATOMIC_FUNC_SUBTRACT(char, uint8_t)
 594 _CSAN_ATOMIC_FUNC_STORE(char, uint8_t)
 595 #if 0
 596 CSAN_ATOMIC_FUNC_FETCHADD(char, uint8_t)
 597 CSAN_ATOMIC_FUNC_READANDCLEAR(char, uint8_t)
 598 CSAN_ATOMIC_FUNC_SWAP(char, uint8_t)
 599 CSAN_ATOMIC_FUNC_TESTANDCLEAR(char, uint8_t)
 600 CSAN_ATOMIC_FUNC_TESTANDSET(char, uint8_t)
 601 #endif
 602
 603 CSAN_ATOMIC_FUNC_ADD(short, uint16_t)
 604 CSAN_ATOMIC_FUNC_CLEAR(short, uint16_t)
 605 CSAN_ATOMIC_FUNC_CMPSET(short, uint16_t)
 606 CSAN_ATOMIC_FUNC_FCMPSET(short, uint16_t)
 607 CSAN_ATOMIC_FUNC_LOAD(short, uint16_t)
 608 CSAN_ATOMIC_FUNC_SET(short, uint16_t)
 609 CSAN_ATOMIC_FUNC_SUBTRACT(short, uint16_t)
 610 _CSAN_ATOMIC_FUNC_STORE(short, uint16_t)
 611 #if 0
 612 CSAN_ATOMIC_FUNC_FETCHADD(short, uint16_t)
 613 CSAN_ATOMIC_FUNC_READANDCLEAR(short, uint16_t)
 614 CSAN_ATOMIC_FUNC_SWAP(short, uint16_t)
 615 CSAN_ATOMIC_FUNC_TESTANDCLEAR(short, uint16_t)
 616 CSAN_ATOMIC_FUNC_TESTANDSET(short, uint16_t)
 617 #endif
 618
 619 CSAN_ATOMIC_FUNC_ADD(int, u_int)
 620 CSAN_ATOMIC_FUNC_CLEAR(int, u_int)
 621 CSAN_ATOMIC_FUNC_CMPSET(int, u_int)
 622 CSAN_ATOMIC_FUNC_FCMPSET(int, u_int)
 623 CSAN_ATOMIC_FUNC_FETCHADD(int, u_int)
 624 CSAN_ATOMIC_FUNC_LOAD(int, u_int)
 625 CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int)
 626 CSAN_ATOMIC_FUNC_SET(int, u_int)
 627 CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int)
 628 CSAN_ATOMIC_FUNC_STORE(int, u_int)
 629 CSAN_ATOMIC_FUNC_SWAP(int, u_int)
 630 CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int)
 631 CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int)
 632
 633 CSAN_ATOMIC_FUNC_ADD(long, u_long)
 634 CSAN_ATOMIC_FUNC_CLEAR(long, u_long)
 635 CSAN_ATOMIC_FUNC_CMPSET(long, u_long)
 636 CSAN_ATOMIC_FUNC_FCMPSET(long, u_long)
 637 CSAN_ATOMIC_FUNC_FETCHADD(long, u_long)
 638 CSAN_ATOMIC_FUNC_LOAD(long, u_long)
 639 CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long)
 640 CSAN_ATOMIC_FUNC_SET(long, u_long)
 641 CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long)
 642 CSAN_ATOMIC_FUNC_STORE(long, u_long)
 643 CSAN_ATOMIC_FUNC_SWAP(long, u_long)
 644 CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long)
 645 CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long)
 646 CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long)
 647
 648 CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t)
 649 CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t)
 650 CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t)
 651 CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t)
 652 #if !defined(__amd64__)
 653 CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t)
 654 #endif
 655 CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t)
 656 CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t)
 657 CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t)
 658 CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t)
 659 CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t)
 660 CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t)
 661 #if 0
 662 CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t)
 663 CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t)
 664 #endif
 665
 666 #define CSAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
 667         void kcsan_atomic_thread_fence_##name(void)                     \
 668         {                                                               \
 669                 atomic_thread_fence_##name();                           \
 670         }
 671
 672 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq)
 673 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel)
 674 CSAN_ATOMIC_FUNC_THREAD_FENCE(rel)
 675 CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst)
 676
 677 void
 678 kcsan_atomic_interrupt_fence(void)
 679 {
 680         atomic_interrupt_fence();
 681 }
 682
 683 /* -------------------------------------------------------------------------- */
 684
 685 #include <sys/bus.h>
 686 #include <machine/bus.h>
 687 #include <sys/bus_san.h>
 688
 689 int
 690 kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
 691     int flags, bus_space_handle_t *handlep)
 692 {
 693
 694         return (bus_space_map(tag, hnd, size, flags, handlep));
 695 }
 696
 697 void
 698 kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
 699     bus_size_t size)
 700 {
 701
 702         bus_space_unmap(tag, hnd, size);
 703 }
 704
 705 int
 706 kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
 707     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
 708 {
 709
 710         return (bus_space_subregion(tag, hnd, offset, size, handlep));
 711 }
 712
 713 #if !defined(__amd64__)
 714 int
 715 kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start,
 716     bus_addr_t reg_end, bus_size_t size, bus_size_t alignment,
 717     bus_size_t boundary, int flags, bus_addr_t *addrp,
 718     bus_space_handle_t *handlep)
 719 {
 720
 721         return (bus_space_alloc(tag, reg_start, reg_end, size, alignment,
 722             boundary, flags, addrp, handlep));
 723 }
 724 #endif
 725
 726 void
 727 kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
 728     bus_size_t size)
 729 {
 730
 731         bus_space_free(tag, hnd, size);
 732 }
 733
 734 void
 735 kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
 736     bus_size_t offset, bus_size_t size, int flags)
 737 {
 738
 739         bus_space_barrier(tag, hnd, offset, size, flags);
 740 }
 741
 742 #define CSAN_BUS_READ_FUNC(func, width, type)                           \
 743         type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
 744             bus_space_handle_t hnd, bus_size_t offset)                  \
 745         {                                                               \
 746                 return (bus_space_read##func##_##width(tag, hnd,        \
 747                     offset));                                           \
 748         }                                                               \
 749
 750 #define CSAN_BUS_READ_PTR_FUNC(func, width, type)                       \
 751         void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
 752             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
 753             bus_size_t count)                                           \
 754         {                                                               \
 755                 kcsan_access((uintptr_t)buf, sizeof(type) * count,      \
 756                     false, false, __RET_ADDR);                          \
 757                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
 758                     count);                                             \
 759         }
 760
 761 CSAN_BUS_READ_FUNC(, 1, uint8_t)
 762 CSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
 763 CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
 764 CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
 765 CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
 766 CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
 767
 768 CSAN_BUS_READ_FUNC(, 2, uint16_t)
 769 CSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
 770 CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
 771 CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
 772 CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
 773 CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
 774
 775 CSAN_BUS_READ_FUNC(, 4, uint32_t)
 776 CSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
 777 CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
 778 CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
 779 CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
 780 CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
 781
 782 CSAN_BUS_READ_FUNC(, 8, uint64_t)
 783 #if defined(__aarch64__)
 784 CSAN_BUS_READ_FUNC(_stream, 8, uint64_t)
 785 CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
 786 CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
 787 CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
 788 CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
 789 #endif
 790
 791 #define CSAN_BUS_WRITE_FUNC(func, width, type)                          \
 792         void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag,         \
 793             bus_space_handle_t hnd, bus_size_t offset, type value)      \
 794         {                                                               \
 795                 bus_space_write##func##_##width(tag, hnd, offset, value); \
 796         }                                                               \
 797
 798 #define CSAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
 799         void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \
 800             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
 801             bus_size_t count)                                           \
 802         {                                                               \
 803                 kcsan_access((uintptr_t)buf, sizeof(type) * count,      \
 804                     true, false, __RET_ADDR);                           \
 805                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
 806                     count);                                             \
 807         }
 808
 809 CSAN_BUS_WRITE_FUNC(, 1, uint8_t)
 810 CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
 811 CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
 812 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
 813 CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
 814 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
 815
 816 CSAN_BUS_WRITE_FUNC(, 2, uint16_t)
 817 CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
 818 CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
 819 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
 820 CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
 821 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
 822
 823 CSAN_BUS_WRITE_FUNC(, 4, uint32_t)
 824 CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
 825 CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
 826 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
 827 CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
 828 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
 829
 830 CSAN_BUS_WRITE_FUNC(, 8, uint64_t)
 831 #if defined(__aarch64__)
 832 CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t)
 833 CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t)
 834 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t)
 835 CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t)
 836 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t)
 837 #endif
 838
 839 #define CSAN_BUS_SET_FUNC(func, width, type)                            \
 840         void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
 841             bus_space_handle_t hnd, bus_size_t offset, type value,      \
 842             bus_size_t count)                                           \
 843         {                                                               \
 844                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
 845                     count);                                             \
 846         }
 847
 848 CSAN_BUS_SET_FUNC(multi, 1, uint8_t)
 849 CSAN_BUS_SET_FUNC(region, 1, uint8_t)
 850 #if !defined(__aarch64__)
 851 CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
 852 CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
 853 #endif
 854
 855 CSAN_BUS_SET_FUNC(multi, 2, uint16_t)
 856 CSAN_BUS_SET_FUNC(region, 2, uint16_t)
 857 #if !defined(__aarch64__)
 858 CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
 859 CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
 860 #endif
 861
 862 CSAN_BUS_SET_FUNC(multi, 4, uint32_t)
 863 CSAN_BUS_SET_FUNC(region, 4, uint32_t)
 864 #if !defined(__aarch64__)
 865 CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
 866 CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
 867 #endif
 868
 869 #if !defined(__amd64__)
 870 CSAN_BUS_SET_FUNC(multi, 8, uint64_t)
 871 CSAN_BUS_SET_FUNC(region, 8, uint64_t)
 872 #if !defined(__aarch64__)
 873 CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t)
 874 CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t)
 875 #endif
 876 #endif
 877
 878 #define CSAN_BUS_PEEK_FUNC(width, type)                                 \
 879         int kcsan_bus_space_peek_##width(bus_space_tag_t tag,           \
 880             bus_space_handle_t hnd, bus_size_t offset, type *value)     \
 881         {                                                               \
 882                 kcsan_access((uintptr_t)value, sizeof(type), true, false, \
 883                     __RET_ADDR);                                        \
 884                 return (bus_space_peek_##width(tag, hnd, offset, value)); \
 885         }
 886
 887 CSAN_BUS_PEEK_FUNC(1, uint8_t)
 888 CSAN_BUS_PEEK_FUNC(2, uint16_t)
 889 CSAN_BUS_PEEK_FUNC(4, uint32_t)
 890 #if !defined(__i386__)
 891 CSAN_BUS_PEEK_FUNC(8, uint64_t)
 892 #endif
 893
 894 #define CSAN_BUS_POKE_FUNC(width, type)                                 \
 895         int kcsan_bus_space_poke_##width(bus_space_tag_t tag,           \
 896             bus_space_handle_t hnd, bus_size_t offset, type value)      \
 897         {                                                               \
 898                 return (bus_space_poke_##width(tag, hnd, offset, value)); \
 899         }
 900
 901 CSAN_BUS_POKE_FUNC(1, uint8_t)
 902 CSAN_BUS_POKE_FUNC(2, uint16_t)
 903 CSAN_BUS_POKE_FUNC(4, uint32_t)
 904 #if !defined(__i386__)
 905 CSAN_BUS_POKE_FUNC(8, uint64_t)
 906 #endif