sys/kern/subr_csan.c

   1 /*      $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $       */
   2
   3 /*
   4  * Copyright (c) 2019 The NetBSD Foundation, Inc.
   5  * All rights reserved.
   6  * Copyright (c) 2019 Andrew Turner
   7  *
   8  * This code is derived from software contributed to The NetBSD Foundation
   9  * by Maxime Villard.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30  * POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #define KCSAN_RUNTIME
  34
  35 #include "opt_ddb.h"
  36
  37 #include <sys/cdefs.h>
  38 __FBSDID("$FreeBSD$");
  39
  40 #include <sys/param.h>
  41 #include <sys/kernel.h>
  42 #include <sys/bus.h>
  43 #include <sys/conf.h>
  44 #include <sys/cpu.h>
  45 #include <sys/csan.h>
  46 #include <sys/proc.h>
  47 #include <sys/smp.h>
  48 #include <sys/systm.h>
  49
  50 #include <ddb/ddb.h>
  51 #include <ddb/db_sym.h>
  52
  53 #ifdef KCSAN_PANIC
  54 #define REPORT panic
  55 #else
  56 #define REPORT printf
  57 #endif
  58
  59 typedef struct {
  60         uintptr_t addr;
  61         uint32_t size;
  62         bool write:1;
  63         bool atomic:1;
  64         uintptr_t pc;
  65 } csan_cell_t;
  66
  67 typedef struct {
  68         bool inited;
  69         uint32_t cnt;
  70         csan_cell_t cell;
  71 } csan_cpu_t;
  72
  73 static csan_cpu_t kcsan_cpus[MAXCPU];
  74 static bool kcsan_enabled __read_mostly;
  75
  76 #define __RET_ADDR      (uintptr_t)__builtin_return_address(0)
  77
  78 #define KCSAN_NACCESSES 1024
  79 #define KCSAN_DELAY     10      /* 10 microseconds */
  80
  81 /* -------------------------------------------------------------------------- */
  82
  83 /* The MD code. */
  84 #include <machine/csan.h>
  85
  86 /* -------------------------------------------------------------------------- */
  87
  88 static void
  89 kcsan_enable(void *dummy __unused)
  90 {
  91
  92         printf("Enabling KCSCAN, expect reduced performance.\n");
  93         kcsan_enabled = true;
  94 }
  95 SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL);
  96
  97 void
  98 kcsan_cpu_init(u_int cpu)
  99 {
 100         kcsan_cpus[cpu].inited = true;
 101 }
 102
 103 /* -------------------------------------------------------------------------- */
 104
 105 static inline void
 106 kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu)
 107 {
 108         const char *newsym, *oldsym;
 109 #ifdef DDB
 110         c_db_sym_t sym;
 111         db_expr_t offset;
 112
 113         sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset);
 114         db_symbol_values(sym, &newsym, NULL);
 115
 116         sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset);
 117         db_symbol_values(sym, &oldsym, NULL);
 118 #else
 119         newsym = "";
 120         oldsym = "";
 121 #endif
 122         REPORT("CSan: Racy Access "
 123             "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] "
 124             "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n",
 125             newcpu,
 126             (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"),
 127             (void *)new->addr, new->size, (void *)new->pc, newsym,
 128             oldcpu,
 129             (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"),
 130             (void *)old->addr, old->size, (void *)old->pc, oldsym);
 131         kcsan_md_unwind();
 132 }
 133
 134 static inline bool
 135 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old)
 136 {
 137         if (new->write && !new->atomic)
 138                 return false;
 139         if (old->write && !old->atomic)
 140                 return false;
 141         return true;
 142 }
 143
 144 static inline void
 145 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc)
 146 {
 147         csan_cell_t old, new;
 148         csan_cpu_t *cpu;
 149         uint64_t intr;
 150         size_t i;
 151
 152         if (__predict_false(!kcsan_enabled))
 153                 return;
 154         if (__predict_false(kcsan_md_unsupported((vm_offset_t)addr)))
 155                 return;
 156         if (KERNEL_PANICKED())
 157                 return;
 158
 159         new.addr = addr;
 160         new.size = size;
 161         new.write = write;
 162         new.atomic = atomic;
 163         new.pc = pc;
 164
 165         CPU_FOREACH(i) {
 166                 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old));
 167
 168                 if (old.addr + old.size <= new.addr)
 169                         continue;
 170                 if (new.addr + new.size <= old.addr)
 171                         continue;
 172                 if (__predict_true(!old.write && !new.write))
 173                         continue;
 174                 if (__predict_true(kcsan_access_is_atomic(&new, &old)))
 175                         continue;
 176
 177                 kcsan_report(&new, PCPU_GET(cpuid), &old, i);
 178                 break;
 179         }
 180
 181         if (__predict_false(!kcsan_md_is_avail()))
 182                 return;
 183
 184         kcsan_md_disable_intrs(&intr);
 185
 186         cpu = &kcsan_cpus[PCPU_GET(cpuid)];
 187         if (__predict_false(!cpu->inited))
 188                 goto out;
 189         cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES;
 190         if (__predict_true(cpu->cnt != 0))
 191                 goto out;
 192
 193         __builtin_memcpy(&cpu->cell, &new, sizeof(new));
 194         kcsan_md_delay(KCSAN_DELAY);
 195         __builtin_memset(&cpu->cell, 0, sizeof(new));
 196
 197 out:
 198         kcsan_md_enable_intrs(&intr);
 199 }
 200
 201 #define CSAN_READ(size)                                                 \
 202         void __tsan_read##size(uintptr_t);                              \
 203         void __tsan_read##size(uintptr_t addr)                          \
 204         {                                                               \
 205                 kcsan_access(addr, size, false, false, __RET_ADDR);     \
 206         }                                                               \
 207         void __tsan_unaligned_read##size(uintptr_t);                    \
 208         void __tsan_unaligned_read##size(uintptr_t addr)                \
 209         {                                                               \
 210                 kcsan_access(addr, size, false, false, __RET_ADDR);     \
 211         }
 212
 213 CSAN_READ(1)
 214 CSAN_READ(2)
 215 CSAN_READ(4)
 216 CSAN_READ(8)
 217 CSAN_READ(16)
 218
 219 #define CSAN_WRITE(size)                                                \
 220         void __tsan_write##size(uintptr_t);                             \
 221         void __tsan_write##size(uintptr_t addr)                         \
 222         {                                                               \
 223                 kcsan_access(addr, size, true, false, __RET_ADDR);      \
 224         }                                                               \
 225         void __tsan_unaligned_write##size(uintptr_t);                   \
 226         void __tsan_unaligned_write##size(uintptr_t addr)               \
 227         {                                                               \
 228                 kcsan_access(addr, size, true, false, __RET_ADDR);      \
 229         }
 230
 231 CSAN_WRITE(1)
 232 CSAN_WRITE(2)
 233 CSAN_WRITE(4)
 234 CSAN_WRITE(8)
 235 CSAN_WRITE(16)
 236
 237 void __tsan_read_range(uintptr_t, size_t);
 238 void __tsan_write_range(uintptr_t, size_t);
 239
 240 void
 241 __tsan_read_range(uintptr_t addr, size_t size)
 242 {
 243         kcsan_access(addr, size, false, false, __RET_ADDR);
 244 }
 245
 246 void
 247 __tsan_write_range(uintptr_t addr, size_t size)
 248 {
 249         kcsan_access(addr, size, true, false, __RET_ADDR);
 250 }
 251
 252 void __tsan_init(void);
 253 void __tsan_func_entry(void *);
 254 void __tsan_func_exit(void);
 255
 256 void
 257 __tsan_init(void)
 258 {
 259 }
 260
 261 void
 262 __tsan_func_entry(void *call_pc)
 263 {
 264 }
 265
 266 void
 267 __tsan_func_exit(void)
 268 {
 269 }
 270
 271 /* -------------------------------------------------------------------------- */
 272
 273 void *
 274 kcsan_memcpy(void *dst, const void *src, size_t len)
 275 {
 276         kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
 277         kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
 278         return __builtin_memcpy(dst, src, len);
 279 }
 280
 281 int
 282 kcsan_memcmp(const void *b1, const void *b2, size_t len)
 283 {
 284         kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR);
 285         kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR);
 286         return __builtin_memcmp(b1, b2, len);
 287 }
 288
 289 void *
 290 kcsan_memset(void *b, int c, size_t len)
 291 {
 292         kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR);
 293         return __builtin_memset(b, c, len);
 294 }
 295
 296 void *
 297 kcsan_memmove(void *dst, const void *src, size_t len)
 298 {
 299         kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
 300         kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
 301         return __builtin_memmove(dst, src, len);
 302 }
 303
 304 char *
 305 kcsan_strcpy(char *dst, const char *src)
 306 {
 307         char *save = dst;
 308
 309         while (1) {
 310                 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR);
 311                 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR);
 312                 *dst = *src;
 313                 if (*src == '\0')
 314                         break;
 315                 src++, dst++;
 316         }
 317
 318         return save;
 319 }
 320
 321 int
 322 kcsan_strcmp(const char *s1, const char *s2)
 323 {
 324         while (1) {
 325                 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR);
 326                 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR);
 327                 if (*s1 != *s2)
 328                         break;
 329                 if (*s1 == '\0')
 330                         return 0;
 331                 s1++, s2++;
 332         }
 333
 334         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
 335 }
 336
 337 size_t
 338 kcsan_strlen(const char *str)
 339 {
 340         const char *s;
 341
 342         s = str;
 343         while (1) {
 344                 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR);
 345                 if (*s == '\0')
 346                         break;
 347                 s++;
 348         }
 349
 350         return (s - str);
 351 }
 352
 353 #undef copyin
 354 #undef copyin_nofault
 355 #undef copyinstr
 356 #undef copyout
 357 #undef copyout_nofault
 358
 359 int
 360 kcsan_copyin(const void *uaddr, void *kaddr, size_t len)
 361 {
 362         kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
 363         return copyin(uaddr, kaddr, len);
 364 }
 365
 366 int
 367 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
 368 {
 369         kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
 370         return copyinstr(uaddr, kaddr, len, done);
 371 }
 372
 373 int
 374 kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
 375 {
 376         kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
 377         return copyout(kaddr, uaddr, len);
 378 }
 379
 380 /* -------------------------------------------------------------------------- */
 381
 382 #include <machine/atomic.h>
 383 #include <sys/_cscan_atomic.h>
 384
 385 #define _CSAN_ATOMIC_FUNC_ADD(name, type)                               \
 386         void kcsan_atomic_add_##name(volatile type *ptr, type val)      \
 387         {                                                               \
 388                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 389                     __RET_ADDR);                                        \
 390                 atomic_add_##name(ptr, val);                            \
 391         }
 392
 393 #define CSAN_ATOMIC_FUNC_ADD(name, type)                                \
 394         _CSAN_ATOMIC_FUNC_ADD(name, type)                               \
 395         _CSAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
 396         _CSAN_ATOMIC_FUNC_ADD(rel_##name, type)
 397
 398 #define _CSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 399         void kcsan_atomic_clear_##name(volatile type *ptr, type val)    \
 400         {                                                               \
 401                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 402                     __RET_ADDR);                                        \
 403                 atomic_clear_##name(ptr, val);                          \
 404         }
 405
 406 #define CSAN_ATOMIC_FUNC_CLEAR(name, type)                              \
 407         _CSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
 408         _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
 409         _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
 410
 411 #define _CSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 412         int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1,   \
 413             type val2)                                                  \
 414         {                                                               \
 415                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 416                     __RET_ADDR);                                        \
 417                 return (atomic_cmpset_##name(ptr, val1, val2));         \
 418         }
 419
 420 #define CSAN_ATOMIC_FUNC_CMPSET(name, type)                             \
 421         _CSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
 422         _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
 423         _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
 424
 425 #define _CSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 426         int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \
 427             type val2)                                                  \
 428         {                                                               \
 429                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 430                     __RET_ADDR);                                        \
 431                 return (atomic_fcmpset_##name(ptr, val1, val2));        \
 432         }
 433
 434 #define CSAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
 435         _CSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
 436         _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
 437         _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
 438
 439 #define CSAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
 440         type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
 441         {                                                               \
 442                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 443                     __RET_ADDR);                                        \
 444                 return (atomic_fetchadd_##name(ptr, val));              \
 445         }
 446
 447 #define _CSAN_ATOMIC_FUNC_LOAD(name, type)                              \
 448         type kcsan_atomic_load_##name(volatile type *ptr)               \
 449         {                                                               \
 450                 kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \
 451                     __RET_ADDR);                                        \
 452                 return (atomic_load_##name(ptr));                       \
 453         }
 454
 455 #define CSAN_ATOMIC_FUNC_LOAD(name, type)                               \
 456         _CSAN_ATOMIC_FUNC_LOAD(name, type)                              \
 457         _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type)                        \
 458
 459 #define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
 460         type kcsan_atomic_readandclear_##name(volatile type *ptr)       \
 461         {                                                               \
 462                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 463                     __RET_ADDR);                                        \
 464                 return (atomic_readandclear_##name(ptr));               \
 465         }
 466
 467 #define _CSAN_ATOMIC_FUNC_SET(name, type)                               \
 468         void kcsan_atomic_set_##name(volatile type *ptr, type val)      \
 469         {                                                               \
 470                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 471                     __RET_ADDR);                                        \
 472                 atomic_set_##name(ptr, val);                            \
 473         }
 474
 475 #define CSAN_ATOMIC_FUNC_SET(name, type)                                \
 476         _CSAN_ATOMIC_FUNC_SET(name, type)                               \
 477         _CSAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
 478         _CSAN_ATOMIC_FUNC_SET(rel_##name, type)
 479
 480 #define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 481         void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \
 482         {                                                               \
 483                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 484                     __RET_ADDR);                                        \
 485                 atomic_subtract_##name(ptr, val);                       \
 486         }
 487
 488 #define CSAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
 489         _CSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
 490         _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
 491         _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
 492
 493 #define _CSAN_ATOMIC_FUNC_STORE(name, type)                             \
 494         void kcsan_atomic_store_##name(volatile type *ptr, type val)    \
 495         {                                                               \
 496                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 497                     __RET_ADDR);                                        \
 498                 atomic_store_##name(ptr, val);                          \
 499         }
 500
 501 #define CSAN_ATOMIC_FUNC_STORE(name, type)                              \
 502         _CSAN_ATOMIC_FUNC_STORE(name, type)                             \
 503         _CSAN_ATOMIC_FUNC_STORE(rel_##name, type)
 504
 505 #define CSAN_ATOMIC_FUNC_SWAP(name, type)                               \
 506         type kcsan_atomic_swap_##name(volatile type *ptr, type val)     \
 507         {                                                               \
 508                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 509                     __RET_ADDR);                                        \
 510                 return(atomic_swap_##name(ptr, val));                   \
 511         }
 512
 513 #define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
 514         int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \
 515         {                                                               \
 516                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 517                     __RET_ADDR);                                        \
 518                 return(atomic_testandclear_##name(ptr, val));           \
 519         }
 520
 521 #define CSAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
 522         int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \
 523         {                                                               \
 524                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
 525                     __RET_ADDR);                                        \
 526                 return (atomic_testandset_##name(ptr, val));            \
 527         }
 528
 529 CSAN_ATOMIC_FUNC_ADD(8, uint8_t)
 530 CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t)
 531 CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t)
 532 CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t)
 533 CSAN_ATOMIC_FUNC_LOAD(8, uint8_t)
 534 CSAN_ATOMIC_FUNC_SET(8, uint8_t)
 535 CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t)
 536 _CSAN_ATOMIC_FUNC_STORE(8, uint8_t)
 537 #if 0
 538 CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t)
 539 CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t)
 540 CSAN_ATOMIC_FUNC_SWAP(8, uint8_t)
 541 CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t)
 542 CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t)
 543 #endif
 544
 545 CSAN_ATOMIC_FUNC_ADD(16, uint16_t)
 546 CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t)
 547 CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t)
 548 CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t)
 549 CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
 550 CSAN_ATOMIC_FUNC_SET(16, uint16_t)
 551 CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t)
 552 _CSAN_ATOMIC_FUNC_STORE(16, uint16_t)
 553 #if 0
 554 CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t)
 555 CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t)
 556 CSAN_ATOMIC_FUNC_SWAP(16, uint16_t)
 557 CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t)
 558 CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t)
 559 #endif
 560
 561 CSAN_ATOMIC_FUNC_ADD(32, uint32_t)
 562 CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t)
 563 CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t)
 564 CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t)
 565 CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t)
 566 CSAN_ATOMIC_FUNC_LOAD(32, uint32_t)
 567 CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t)
 568 CSAN_ATOMIC_FUNC_SET(32, uint32_t)
 569 CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t)
 570 CSAN_ATOMIC_FUNC_STORE(32, uint32_t)
 571 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t)
 572 #if !defined(__aarch64__)
 573 CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t)
 574 CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t)
 575 #endif
 576
 577 CSAN_ATOMIC_FUNC_ADD(64, uint64_t)
 578 CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t)
 579 CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t)
 580 CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t)
 581 CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t)
 582 CSAN_ATOMIC_FUNC_LOAD(64, uint64_t)
 583 CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t)
 584 CSAN_ATOMIC_FUNC_SET(64, uint64_t)
 585 CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t)
 586 CSAN_ATOMIC_FUNC_STORE(64, uint64_t)
 587 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t)
 588 #if !defined(__aarch64__)
 589 CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t)
 590 CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t)
 591 #endif
 592
 593 CSAN_ATOMIC_FUNC_ADD(char, uint8_t)
 594 CSAN_ATOMIC_FUNC_CLEAR(char, uint8_t)
 595 CSAN_ATOMIC_FUNC_CMPSET(char, uint8_t)
 596 CSAN_ATOMIC_FUNC_FCMPSET(char, uint8_t)
 597 CSAN_ATOMIC_FUNC_LOAD(char, uint8_t)
 598 CSAN_ATOMIC_FUNC_SET(char, uint8_t)
 599 CSAN_ATOMIC_FUNC_SUBTRACT(char, uint8_t)
 600 _CSAN_ATOMIC_FUNC_STORE(char, uint8_t)
 601 #if 0
 602 CSAN_ATOMIC_FUNC_FETCHADD(char, uint8_t)
 603 CSAN_ATOMIC_FUNC_READANDCLEAR(char, uint8_t)
 604 CSAN_ATOMIC_FUNC_SWAP(char, uint8_t)
 605 CSAN_ATOMIC_FUNC_TESTANDCLEAR(char, uint8_t)
 606 CSAN_ATOMIC_FUNC_TESTANDSET(char, uint8_t)
 607 #endif
 608
 609 CSAN_ATOMIC_FUNC_ADD(short, uint16_t)
 610 CSAN_ATOMIC_FUNC_CLEAR(short, uint16_t)
 611 CSAN_ATOMIC_FUNC_CMPSET(short, uint16_t)
 612 CSAN_ATOMIC_FUNC_FCMPSET(short, uint16_t)
 613 CSAN_ATOMIC_FUNC_LOAD(short, uint16_t)
 614 CSAN_ATOMIC_FUNC_SET(short, uint16_t)
 615 CSAN_ATOMIC_FUNC_SUBTRACT(short, uint16_t)
 616 _CSAN_ATOMIC_FUNC_STORE(short, uint16_t)
 617 #if 0
 618 CSAN_ATOMIC_FUNC_FETCHADD(short, uint16_t)
 619 CSAN_ATOMIC_FUNC_READANDCLEAR(short, uint16_t)
 620 CSAN_ATOMIC_FUNC_SWAP(short, uint16_t)
 621 CSAN_ATOMIC_FUNC_TESTANDCLEAR(short, uint16_t)
 622 CSAN_ATOMIC_FUNC_TESTANDSET(short, uint16_t)
 623 #endif
 624
 625 CSAN_ATOMIC_FUNC_ADD(int, u_int)
 626 CSAN_ATOMIC_FUNC_CLEAR(int, u_int)
 627 CSAN_ATOMIC_FUNC_CMPSET(int, u_int)
 628 CSAN_ATOMIC_FUNC_FCMPSET(int, u_int)
 629 CSAN_ATOMIC_FUNC_FETCHADD(int, u_int)
 630 CSAN_ATOMIC_FUNC_LOAD(int, u_int)
 631 CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int)
 632 CSAN_ATOMIC_FUNC_SET(int, u_int)
 633 CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int)
 634 CSAN_ATOMIC_FUNC_STORE(int, u_int)
 635 CSAN_ATOMIC_FUNC_SWAP(int, u_int)
 636 #if !defined(__aarch64__)
 637 CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int)
 638 CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int)
 639 #endif
 640
 641 CSAN_ATOMIC_FUNC_ADD(long, u_long)
 642 CSAN_ATOMIC_FUNC_CLEAR(long, u_long)
 643 CSAN_ATOMIC_FUNC_CMPSET(long, u_long)
 644 CSAN_ATOMIC_FUNC_FCMPSET(long, u_long)
 645 CSAN_ATOMIC_FUNC_FETCHADD(long, u_long)
 646 CSAN_ATOMIC_FUNC_LOAD(long, u_long)
 647 CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long)
 648 CSAN_ATOMIC_FUNC_SET(long, u_long)
 649 CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long)
 650 CSAN_ATOMIC_FUNC_STORE(long, u_long)
 651 CSAN_ATOMIC_FUNC_SWAP(long, u_long)
 652 #if !defined(__aarch64__)
 653 CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long)
 654 CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long)
 655 CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long)
 656 #endif
 657
 658 CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t)
 659 CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t)
 660 CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t)
 661 CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t)
 662 #if !defined(__amd64__)
 663 CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t)
 664 #endif
 665 CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t)
 666 CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t)
 667 CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t)
 668 CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t)
 669 CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t)
 670 CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t)
 671 #if 0
 672 CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t)
 673 CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t)
 674 #endif
 675
 676 #define CSAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
 677         void kcsan_atomic_thread_fence_##name(void)                     \
 678         {                                                               \
 679                 atomic_thread_fence_##name();                           \
 680         }
 681
 682 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq)
 683 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel)
 684 CSAN_ATOMIC_FUNC_THREAD_FENCE(rel)
 685 CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst)
 686
 687 /* -------------------------------------------------------------------------- */
 688
 689 #include <sys/bus.h>
 690 #include <machine/bus.h>
 691 #include <sys/_cscan_bus.h>
 692
 693 int
 694 kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
 695     int flags, bus_space_handle_t *handlep)
 696 {
 697
 698         return (bus_space_map(tag, hnd, size, flags, handlep));
 699 }
 700
 701 void
 702 kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
 703     bus_size_t size)
 704 {
 705
 706         bus_space_unmap(tag, hnd, size);
 707 }
 708
 709 int
 710 kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
 711     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
 712 {
 713
 714         return (bus_space_subregion(tag, hnd, offset, size, handlep));
 715 }
 716
 717 #if !defined(__amd64__)
 718 int
 719 kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start,
 720     bus_addr_t reg_end, bus_size_t size, bus_size_t alignment,
 721     bus_size_t boundary, int flags, bus_addr_t *addrp,
 722     bus_space_handle_t *handlep)
 723 {
 724
 725         return (bus_space_alloc(tag, reg_start, reg_end, size, alignment,
 726             boundary, flags, addrp, handlep));
 727 }
 728 #endif
 729
 730 void
 731 kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
 732     bus_size_t size)
 733 {
 734
 735         bus_space_free(tag, hnd, size);
 736 }
 737
 738 void
 739 kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
 740     bus_size_t offset, bus_size_t size, int flags)
 741 {
 742
 743         bus_space_barrier(tag, hnd, offset, size, flags);
 744 }
 745
 746 #define CSAN_BUS_READ_FUNC(func, width, type)                           \
 747         type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
 748             bus_space_handle_t hnd, bus_size_t offset)                  \
 749         {                                                               \
 750                 return (bus_space_read##func##_##width(tag, hnd,        \
 751                     offset));                                           \
 752         }                                                               \
 753
 754 #define CSAN_BUS_READ_PTR_FUNC(func, width, type)                       \
 755         void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
 756             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
 757             bus_size_t count)                                           \
 758         {                                                               \
 759                 kcsan_access((uintptr_t)buf, sizeof(type) * count,      \
 760                     false, false, __RET_ADDR);                          \
 761                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
 762                     count);                                             \
 763         }
 764
 765 CSAN_BUS_READ_FUNC(, 1, uint8_t)
 766 CSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
 767 CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
 768 CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
 769 CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
 770 CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
 771
 772 CSAN_BUS_READ_FUNC(, 2, uint16_t)
 773 CSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
 774 CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
 775 CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
 776 CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
 777 CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
 778
 779 CSAN_BUS_READ_FUNC(, 4, uint32_t)
 780 CSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
 781 CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
 782 CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
 783 CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
 784 CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
 785
 786 CSAN_BUS_READ_FUNC(, 8, uint64_t)
 787 #if defined(__aarch64__)
 788 CSAN_BUS_READ_FUNC(_stream, 8, uint64_t)
 789 CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
 790 CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
 791 CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
 792 CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
 793 #endif
 794
 795 #define CSAN_BUS_WRITE_FUNC(func, width, type)                          \
 796         void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag,         \
 797             bus_space_handle_t hnd, bus_size_t offset, type value)      \
 798         {                                                               \
 799                 bus_space_write##func##_##width(tag, hnd, offset, value); \
 800         }                                                               \
 801
 802 #define CSAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
 803         void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \
 804             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
 805             bus_size_t count)                                           \
 806         {                                                               \
 807                 kcsan_access((uintptr_t)buf, sizeof(type) * count,      \
 808                     true, false, __RET_ADDR);                           \
 809                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
 810                     count);                                             \
 811         }
 812
 813 CSAN_BUS_WRITE_FUNC(, 1, uint8_t)
 814 CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
 815 CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
 816 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
 817 CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
 818 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
 819
 820 CSAN_BUS_WRITE_FUNC(, 2, uint16_t)
 821 CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
 822 CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
 823 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
 824 CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
 825 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
 826
 827 CSAN_BUS_WRITE_FUNC(, 4, uint32_t)
 828 CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
 829 CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
 830 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
 831 CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
 832 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
 833
 834 CSAN_BUS_WRITE_FUNC(, 8, uint64_t)
 835 #if defined(__aarch64__)
 836 CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t)
 837 CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t)
 838 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t)
 839 CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t)
 840 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t)
 841 #endif
 842
 843 #define CSAN_BUS_SET_FUNC(func, width, type)                            \
 844         void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
 845             bus_space_handle_t hnd, bus_size_t offset, type value,      \
 846             bus_size_t count)                                           \
 847         {                                                               \
 848                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
 849                     count);                                             \
 850         }
 851
 852 CSAN_BUS_SET_FUNC(multi, 1, uint8_t)
 853 CSAN_BUS_SET_FUNC(region, 1, uint8_t)
 854 #if !defined(__aarch64__)
 855 CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
 856 CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
 857 #endif
 858
 859 CSAN_BUS_SET_FUNC(multi, 2, uint16_t)
 860 CSAN_BUS_SET_FUNC(region, 2, uint16_t)
 861 #if !defined(__aarch64__)
 862 CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
 863 CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
 864 #endif
 865
 866 CSAN_BUS_SET_FUNC(multi, 4, uint32_t)
 867 CSAN_BUS_SET_FUNC(region, 4, uint32_t)
 868 #if !defined(__aarch64__)
 869 CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
 870 CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
 871 #endif
 872
 873 #if !defined(__amd64__)
 874 CSAN_BUS_SET_FUNC(multi, 8, uint64_t)
 875 CSAN_BUS_SET_FUNC(region, 8, uint64_t)
 876 #if !defined(__aarch64__)
 877 CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t)
 878 CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t)
 879 #endif
 880 #endif
 881