]> CyberLeo.Net >> Repos - FreeBSD/FreeBSD.git/blob - sys/kern/subr_csan.c
Port the NetBSD KCSAN runtime to FreeBSD.
[FreeBSD/FreeBSD.git] / sys / kern / subr_csan.c
1 /*      $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $       */
2
3 /*
4  * Copyright (c) 2019 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  * Copyright (c) 2019 Andrew Turner
7  *
8  * This code is derived from software contributed to The NetBSD Foundation
9  * by Maxime Villard.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  * POSSIBILITY OF SUCH DAMAGE.
31  */
32
33 #define KCSAN_RUNTIME
34
35 #include "opt_ddb.h"
36
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39
40 #include <sys/param.h>
41 #include <sys/kernel.h>
42 #include <sys/bus.h>
43 #include <sys/conf.h>
44 #include <sys/cpu.h>
45 #include <sys/csan.h>
46 #include <sys/proc.h>
47 #include <sys/smp.h>
48 #include <sys/systm.h>
49
50 #include <ddb/ddb.h>
51 #include <ddb/db_sym.h>
52
53 #ifdef KCSAN_PANIC
54 #define REPORT panic
55 #else
56 #define REPORT printf
57 #endif
58
59 typedef struct {
60         uintptr_t addr;
61         uint32_t size;
62         bool write:1;
63         bool atomic:1;
64         uintptr_t pc;
65 } csan_cell_t;
66
67 typedef struct {
68         bool inited;
69         uint32_t cnt;
70         csan_cell_t cell;
71 } csan_cpu_t;
72
73 static csan_cpu_t kcsan_cpus[MAXCPU];
74 static bool kcsan_enabled __read_mostly;
75
76 #define __RET_ADDR      (uintptr_t)__builtin_return_address(0)
77
78 #define KCSAN_NACCESSES 1024
79 #define KCSAN_DELAY     10      /* 10 microseconds */
80
81 /* -------------------------------------------------------------------------- */
82
83 /* The MD code. */
84 #include <machine/csan.h>
85
86 /* -------------------------------------------------------------------------- */
87
88 static void
89 kcsan_enable(void *dummy __unused)
90 {
91
92         printf("Enabling KCSCAN, expect reduced performance.\n");
93         kcsan_enabled = true;
94 }
95 SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL);
96
97 void
98 kcsan_cpu_init(u_int cpu)
99 {
100         kcsan_cpus[cpu].inited = true;
101 }
102
103 /* -------------------------------------------------------------------------- */
104
105 static inline void
106 kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu)
107 {
108         const char *newsym, *oldsym;
109 #ifdef DDB
110         c_db_sym_t sym;
111         db_expr_t offset;
112
113         sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset);
114         db_symbol_values(sym, &newsym, NULL);
115
116         sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset);
117         db_symbol_values(sym, &oldsym, NULL);
118 #else
119         newsym = "";
120         oldsym = "";
121 #endif
122         REPORT("CSan: Racy Access "
123             "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] "
124             "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n",
125             newcpu,
126             (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"),
127             (void *)new->addr, new->size, (void *)new->pc, newsym,
128             oldcpu,
129             (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"),
130             (void *)old->addr, old->size, (void *)old->pc, oldsym);
131         kcsan_md_unwind();
132 }
133
134 static inline bool
135 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old)
136 {
137         if (new->write && !new->atomic)
138                 return false;
139         if (old->write && !old->atomic)
140                 return false;
141         return true;
142 }
143
144 static inline void
145 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc)
146 {
147         csan_cell_t old, new;
148         csan_cpu_t *cpu;
149         uint64_t intr;
150         size_t i;
151
152         if (__predict_false(!kcsan_enabled))
153                 return;
154
155         new.addr = addr;
156         new.size = size;
157         new.write = write;
158         new.atomic = atomic;
159         new.pc = pc;
160
161         CPU_FOREACH(i) {
162                 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old));
163
164                 if (old.addr + old.size <= new.addr)
165                         continue;
166                 if (new.addr + new.size <= old.addr)
167                         continue;
168                 if (__predict_true(!old.write && !new.write))
169                         continue;
170                 if (__predict_true(kcsan_access_is_atomic(&new, &old)))
171                         continue;
172
173                 kcsan_report(&new, PCPU_GET(cpuid), &old, i);
174                 break;
175         }
176
177         if (__predict_false(!kcsan_md_is_avail()))
178                 return;
179
180         kcsan_md_disable_intrs(&intr);
181
182         cpu = &kcsan_cpus[PCPU_GET(cpuid)];
183         if (__predict_false(!cpu->inited))
184                 goto out;
185         cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES;
186         if (__predict_true(cpu->cnt != 0))
187                 goto out;
188
189         __builtin_memcpy(&cpu->cell, &new, sizeof(new));
190         kcsan_md_delay(KCSAN_DELAY);
191         __builtin_memset(&cpu->cell, 0, sizeof(new));
192
193 out:
194         kcsan_md_enable_intrs(&intr);
195 }
196
197 #define CSAN_READ(size)                                                 \
198         void __tsan_read##size(uintptr_t);                              \
199         void __tsan_read##size(uintptr_t addr)                          \
200         {                                                               \
201                 kcsan_access(addr, size, false, false, __RET_ADDR);     \
202         }                                                               \
203         void __tsan_unaligned_read##size(uintptr_t);                    \
204         void __tsan_unaligned_read##size(uintptr_t addr)                \
205         {                                                               \
206                 kcsan_access(addr, size, false, false, __RET_ADDR);     \
207         }
208
209 CSAN_READ(1)
210 CSAN_READ(2)
211 CSAN_READ(4)
212 CSAN_READ(8)
213 CSAN_READ(16)
214
215 #define CSAN_WRITE(size)                                                \
216         void __tsan_write##size(uintptr_t);                             \
217         void __tsan_write##size(uintptr_t addr)                         \
218         {                                                               \
219                 kcsan_access(addr, size, true, false, __RET_ADDR);      \
220         }                                                               \
221         void __tsan_unaligned_write##size(uintptr_t);                   \
222         void __tsan_unaligned_write##size(uintptr_t addr)               \
223         {                                                               \
224                 kcsan_access(addr, size, true, false, __RET_ADDR);      \
225         }
226
227 CSAN_WRITE(1)
228 CSAN_WRITE(2)
229 CSAN_WRITE(4)
230 CSAN_WRITE(8)
231 CSAN_WRITE(16)
232
233 void __tsan_read_range(uintptr_t, size_t);
234 void __tsan_write_range(uintptr_t, size_t);
235
236 void
237 __tsan_read_range(uintptr_t addr, size_t size)
238 {
239         kcsan_access(addr, size, false, false, __RET_ADDR);
240 }
241
242 void
243 __tsan_write_range(uintptr_t addr, size_t size)
244 {
245         kcsan_access(addr, size, true, false, __RET_ADDR);
246 }
247
248 void __tsan_init(void);
249 void __tsan_func_entry(void *);
250 void __tsan_func_exit(void);
251
252 void
253 __tsan_init(void)
254 {
255 }
256
257 void
258 __tsan_func_entry(void *call_pc)
259 {
260 }
261
262 void
263 __tsan_func_exit(void)
264 {
265 }
266
267 /* -------------------------------------------------------------------------- */
268
269 void *
270 kcsan_memcpy(void *dst, const void *src, size_t len)
271 {
272         kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
273         kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
274         return __builtin_memcpy(dst, src, len);
275 }
276
277 int
278 kcsan_memcmp(const void *b1, const void *b2, size_t len)
279 {
280         kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR);
281         kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR);
282         return __builtin_memcmp(b1, b2, len);
283 }
284
285 void *
286 kcsan_memset(void *b, int c, size_t len)
287 {
288         kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR);
289         return __builtin_memset(b, c, len);
290 }
291
292 void *
293 kcsan_memmove(void *dst, const void *src, size_t len)
294 {
295         kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
296         kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
297         return __builtin_memmove(dst, src, len);
298 }
299
300 char *
301 kcsan_strcpy(char *dst, const char *src)
302 {
303         char *save = dst;
304
305         while (1) {
306                 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR);
307                 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR);
308                 *dst = *src;
309                 if (*src == '\0')
310                         break;
311                 src++, dst++;
312         }
313
314         return save;
315 }
316
317 int
318 kcsan_strcmp(const char *s1, const char *s2)
319 {
320         while (1) {
321                 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR);
322                 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR);
323                 if (*s1 != *s2)
324                         break;
325                 if (*s1 == '\0')
326                         return 0;
327                 s1++, s2++;
328         }
329
330         return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
331 }
332
333 size_t
334 kcsan_strlen(const char *str)
335 {
336         const char *s;
337
338         s = str;
339         while (1) {
340                 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR);
341                 if (*s == '\0')
342                         break;
343                 s++;
344         }
345
346         return (s - str);
347 }
348
349 #undef copystr
350 #undef copyin
351 #undef copyin_nofault
352 #undef copyinstr
353 #undef copyout
354 #undef copyout_nofault
355
356 int
357 kcsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done)
358 {
359         kcsan_access((uintptr_t)kdaddr, len, true, false, __RET_ADDR);
360         return copystr(kfaddr, kdaddr, len, done);
361 }
362
363 int
364 kcsan_copyin(const void *uaddr, void *kaddr, size_t len)
365 {
366         kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
367         return copyin(uaddr, kaddr, len);
368 }
369
370 int
371 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
372 {
373         kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
374         return copyinstr(uaddr, kaddr, len, done);
375 }
376
377 int
378 kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
379 {
380         kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
381         return copyout(kaddr, uaddr, len);
382 }
383
384 /* -------------------------------------------------------------------------- */
385
386 #include <machine/atomic.h>
387 #include <sys/_cscan_atomic.h>
388
389 #define _CSAN_ATOMIC_FUNC_ADD(name, type)                               \
390         void kcsan_atomic_add_##name(volatile type *ptr, type val)      \
391         {                                                               \
392                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
393                     __RET_ADDR);                                        \
394                 atomic_add_##name(ptr, val);                            \
395         }
396
397 #define CSAN_ATOMIC_FUNC_ADD(name, type)                                \
398         _CSAN_ATOMIC_FUNC_ADD(name, type)                               \
399         _CSAN_ATOMIC_FUNC_ADD(acq_##name, type)                         \
400         _CSAN_ATOMIC_FUNC_ADD(rel_##name, type)
401
402 #define _CSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
403         void kcsan_atomic_clear_##name(volatile type *ptr, type val)    \
404         {                                                               \
405                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
406                     __RET_ADDR);                                        \
407                 atomic_clear_##name(ptr, val);                          \
408         }
409
410 #define CSAN_ATOMIC_FUNC_CLEAR(name, type)                              \
411         _CSAN_ATOMIC_FUNC_CLEAR(name, type)                             \
412         _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type)                       \
413         _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
414
415 #define _CSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
416         int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1,   \
417             type val2)                                                  \
418         {                                                               \
419                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
420                     __RET_ADDR);                                        \
421                 return (atomic_cmpset_##name(ptr, val1, val2));         \
422         }
423
424 #define CSAN_ATOMIC_FUNC_CMPSET(name, type)                             \
425         _CSAN_ATOMIC_FUNC_CMPSET(name, type)                            \
426         _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type)                      \
427         _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
428
429 #define _CSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
430         int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \
431             type val2)                                                  \
432         {                                                               \
433                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
434                     __RET_ADDR);                                        \
435                 return (atomic_fcmpset_##name(ptr, val1, val2));        \
436         }
437
438 #define CSAN_ATOMIC_FUNC_FCMPSET(name, type)                            \
439         _CSAN_ATOMIC_FUNC_FCMPSET(name, type)                           \
440         _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type)                     \
441         _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
442
443 #define CSAN_ATOMIC_FUNC_FETCHADD(name, type)                           \
444         type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
445         {                                                               \
446                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
447                     __RET_ADDR);                                        \
448                 return (atomic_fetchadd_##name(ptr, val));              \
449         }
450
451 #define _CSAN_ATOMIC_FUNC_LOAD(name, type)                              \
452         type kcsan_atomic_load_##name(volatile type *ptr)               \
453         {                                                               \
454                 kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \
455                     __RET_ADDR);                                        \
456                 return (atomic_load_##name(ptr));                       \
457         }
458
459 #define CSAN_ATOMIC_FUNC_LOAD(name, type)                               \
460         _CSAN_ATOMIC_FUNC_LOAD(name, type)                              \
461         _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type)                        \
462
463 #define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type)                       \
464         type kcsan_atomic_readandclear_##name(volatile type *ptr)       \
465         {                                                               \
466                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
467                     __RET_ADDR);                                        \
468                 return (atomic_readandclear_##name(ptr));               \
469         }
470
471 #define _CSAN_ATOMIC_FUNC_SET(name, type)                               \
472         void kcsan_atomic_set_##name(volatile type *ptr, type val)      \
473         {                                                               \
474                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
475                     __RET_ADDR);                                        \
476                 atomic_set_##name(ptr, val);                            \
477         }
478
479 #define CSAN_ATOMIC_FUNC_SET(name, type)                                \
480         _CSAN_ATOMIC_FUNC_SET(name, type)                               \
481         _CSAN_ATOMIC_FUNC_SET(acq_##name, type)                         \
482         _CSAN_ATOMIC_FUNC_SET(rel_##name, type)
483
484 #define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
485         void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \
486         {                                                               \
487                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
488                     __RET_ADDR);                                        \
489                 atomic_subtract_##name(ptr, val);                       \
490         }
491
492
493 #define CSAN_ATOMIC_FUNC_SUBTRACT(name, type)                           \
494         _CSAN_ATOMIC_FUNC_SUBTRACT(name, type)                          \
495         _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type)                    \
496         _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
497
498 #define _CSAN_ATOMIC_FUNC_STORE(name, type)                             \
499         void kcsan_atomic_store_##name(volatile type *ptr, type val)    \
500         {                                                               \
501                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
502                     __RET_ADDR);                                        \
503                 atomic_store_##name(ptr, val);                          \
504         }
505
506 #define CSAN_ATOMIC_FUNC_STORE(name, type)                              \
507         _CSAN_ATOMIC_FUNC_STORE(name, type)                             \
508         _CSAN_ATOMIC_FUNC_STORE(rel_##name, type)
509
510 #define CSAN_ATOMIC_FUNC_SWAP(name, type)                               \
511         type kcsan_atomic_swap_##name(volatile type *ptr, type val)     \
512         {                                                               \
513                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
514                     __RET_ADDR);                                        \
515                 return(atomic_swap_##name(ptr, val));                   \
516         }
517
518 #define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type)                       \
519         int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \
520         {                                                               \
521                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
522                     __RET_ADDR);                                        \
523                 return(atomic_testandclear_##name(ptr, val));           \
524         }
525
526 #define CSAN_ATOMIC_FUNC_TESTANDSET(name, type)                         \
527         int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \
528         {                                                               \
529                 kcsan_access((uintptr_t)ptr, sizeof(type), true, true,  \
530                     __RET_ADDR);                                        \
531                 return (atomic_testandset_##name(ptr, val));            \
532         }
533
534
535 CSAN_ATOMIC_FUNC_ADD(8, uint8_t)
536 CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t)
537 CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t)
538 CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t)
539 _CSAN_ATOMIC_FUNC_LOAD(8, uint8_t)
540 CSAN_ATOMIC_FUNC_SET(8, uint8_t)
541 CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t)
542 _CSAN_ATOMIC_FUNC_STORE(8, uint8_t)
543 #if 0
544 CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t)
545 CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t)
546 CSAN_ATOMIC_FUNC_SWAP(8, uint8_t)
547 CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t)
548 CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t)
549 #endif
550
551 CSAN_ATOMIC_FUNC_ADD(16, uint16_t)
552 CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t)
553 CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t)
554 CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t)
555 #if defined(__aarch64__)
556 _CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
557 #else
558 CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
559 #endif
560 CSAN_ATOMIC_FUNC_SET(16, uint16_t)
561 CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t)
562 _CSAN_ATOMIC_FUNC_STORE(16, uint16_t)
563 #if 0
564 CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t)
565 CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t)
566 CSAN_ATOMIC_FUNC_SWAP(16, uint16_t)
567 CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t)
568 CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t)
569 #endif
570
571 CSAN_ATOMIC_FUNC_ADD(32, uint32_t)
572 CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t)
573 CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t)
574 CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t)
575 CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t)
576 CSAN_ATOMIC_FUNC_LOAD(32, uint32_t)
577 CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t)
578 CSAN_ATOMIC_FUNC_SET(32, uint32_t)
579 CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t)
580 CSAN_ATOMIC_FUNC_STORE(32, uint32_t)
581 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t)
582 #if !defined(__aarch64__)
583 CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t)
584 CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t)
585 #endif
586
587 CSAN_ATOMIC_FUNC_ADD(64, uint64_t)
588 CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t)
589 CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t)
590 CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t)
591 CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t)
592 CSAN_ATOMIC_FUNC_LOAD(64, uint64_t)
593 CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t)
594 CSAN_ATOMIC_FUNC_SET(64, uint64_t)
595 CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t)
596 CSAN_ATOMIC_FUNC_STORE(64, uint64_t)
597 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t)
598 #if !defined(__aarch64__)
599 CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t)
600 CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t)
601 #endif
602
603 CSAN_ATOMIC_FUNC_ADD(int, u_int)
604 CSAN_ATOMIC_FUNC_CLEAR(int, u_int)
605 CSAN_ATOMIC_FUNC_CMPSET(int, u_int)
606 CSAN_ATOMIC_FUNC_FCMPSET(int, u_int)
607 CSAN_ATOMIC_FUNC_FETCHADD(int, u_int)
608 CSAN_ATOMIC_FUNC_LOAD(int, u_int)
609 CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int)
610 CSAN_ATOMIC_FUNC_SET(int, u_int)
611 CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int)
612 CSAN_ATOMIC_FUNC_STORE(int, u_int)
613 CSAN_ATOMIC_FUNC_SWAP(int, u_int)
614 #if !defined(__aarch64__)
615 CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int)
616 CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int)
617 #endif
618
619 CSAN_ATOMIC_FUNC_ADD(long, u_long)
620 CSAN_ATOMIC_FUNC_CLEAR(long, u_long)
621 CSAN_ATOMIC_FUNC_CMPSET(long, u_long)
622 CSAN_ATOMIC_FUNC_FCMPSET(long, u_long)
623 CSAN_ATOMIC_FUNC_FETCHADD(long, u_long)
624 CSAN_ATOMIC_FUNC_LOAD(long, u_long)
625 CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long)
626 CSAN_ATOMIC_FUNC_SET(long, u_long)
627 CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long)
628 CSAN_ATOMIC_FUNC_STORE(long, u_long)
629 CSAN_ATOMIC_FUNC_SWAP(long, u_long)
630 #if !defined(__aarch64__)
631 CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long)
632 CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long)
633 #endif
634
635 CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t)
636 CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t)
637 CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t)
638 CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t)
639 #if !defined(__amd64__)
640 CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t)
641 #endif
642 CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t)
643 CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t)
644 CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t)
645 CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t)
646 CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t)
647 CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t)
648 #if 0
649 CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t)
650 CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t)
651 #endif
652
653 #define CSAN_ATOMIC_FUNC_THREAD_FENCE(name)                             \
654         void kcsan_atomic_thread_fence_##name(void)                     \
655         {                                                               \
656                 atomic_thread_fence_##name();                           \
657         }
658
659
660 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq)
661 CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel)
662 CSAN_ATOMIC_FUNC_THREAD_FENCE(rel)
663 CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst)
664
665 /* -------------------------------------------------------------------------- */
666
667 #include <sys/bus.h>
668 #include <machine/bus.h>
669 #include <sys/_cscan_bus.h>
670
671 int
672 kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
673     int flags, bus_space_handle_t *handlep)
674 {
675
676         return (bus_space_map(tag, hnd, size, flags, handlep));
677 }
678
679 void
680 kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
681     bus_size_t size)
682 {
683
684         bus_space_unmap(tag, hnd, size);
685 }
686
687 int
688 kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
689     bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
690 {
691
692         return (bus_space_subregion(tag, hnd, offset, size, handlep));
693 }
694
695 #if !defined(__amd64__)
696 int
697 kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start,
698     bus_addr_t reg_end, bus_size_t size, bus_size_t alignment,
699     bus_size_t boundary, int flags, bus_addr_t *addrp,
700     bus_space_handle_t *handlep)
701 {
702
703         return (bus_space_alloc(tag, reg_start, reg_end, size, alignment,
704             boundary, flags, addrp, handlep));
705 }
706 #endif
707
708 void
709 kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
710     bus_size_t size)
711 {
712
713         bus_space_free(tag, hnd, size);
714 }
715
716 void
717 kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
718     bus_size_t offset, bus_size_t size, int flags)
719 {
720
721         bus_space_barrier(tag, hnd, offset, size, flags);
722 }
723
724 #define CSAN_BUS_READ_FUNC(func, width, type)                           \
725         type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag,  \
726             bus_space_handle_t hnd, bus_size_t offset)                  \
727         {                                                               \
728                 return (bus_space_read##func##_##width(tag, hnd,        \
729                     offset));                                           \
730         }                                                               \
731
732 #define CSAN_BUS_READ_PTR_FUNC(func, width, type)                       \
733         void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
734             bus_space_handle_t hnd, bus_size_t size, type *buf,         \
735             bus_size_t count)                                           \
736         {                                                               \
737                 kcsan_access((uintptr_t)buf, sizeof(type) * count,      \
738                     false, false, __RET_ADDR);                          \
739                 bus_space_read_##func##_##width(tag, hnd, size, buf,    \
740                     count);                                             \
741         }
742
743 CSAN_BUS_READ_FUNC(, 1, uint8_t)
744 CSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
745 CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
746 CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
747 CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
748 CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
749
750 CSAN_BUS_READ_FUNC(, 2, uint16_t)
751 CSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
752 CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
753 CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
754 CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
755 CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
756
757 CSAN_BUS_READ_FUNC(, 4, uint32_t)
758 CSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
759 CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
760 CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
761 CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
762 CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
763
764 CSAN_BUS_READ_FUNC(, 8, uint64_t)
765 #if defined(__aarch64__)
766 CSAN_BUS_READ_FUNC(_stream, 8, uint64_t)
767 CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
768 CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
769 CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
770 CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
771 #endif
772
773 #define CSAN_BUS_WRITE_FUNC(func, width, type)                          \
774         void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag,         \
775             bus_space_handle_t hnd, bus_size_t offset, type value)      \
776         {                                                               \
777                 bus_space_write##func##_##width(tag, hnd, offset, value); \
778         }                                                               \
779
780 #define CSAN_BUS_WRITE_PTR_FUNC(func, width, type)                      \
781         void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \
782             bus_space_handle_t hnd, bus_size_t size, const type *buf,   \
783             bus_size_t count)                                           \
784         {                                                               \
785                 kcsan_access((uintptr_t)buf, sizeof(type) * count,      \
786                     true, false, __RET_ADDR);                           \
787                 bus_space_write_##func##_##width(tag, hnd, size, buf,   \
788                     count);                                             \
789         }
790
791 CSAN_BUS_WRITE_FUNC(, 1, uint8_t)
792 CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
793 CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
794 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
795 CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
796 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
797
798 CSAN_BUS_WRITE_FUNC(, 2, uint16_t)
799 CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
800 CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
801 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
802 CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
803 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
804
805 CSAN_BUS_WRITE_FUNC(, 4, uint32_t)
806 CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
807 CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
808 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
809 CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
810 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
811
812 CSAN_BUS_WRITE_FUNC(, 8, uint64_t)
813 #if defined(__aarch64__)
814 CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t)
815 CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t)
816 CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t)
817 CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t)
818 CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t)
819 #endif
820
821 #define CSAN_BUS_SET_FUNC(func, width, type)                            \
822         void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag,  \
823             bus_space_handle_t hnd, bus_size_t offset, type value,      \
824             bus_size_t count)                                           \
825         {                                                               \
826                 bus_space_set_##func##_##width(tag, hnd, offset, value, \
827                     count);                                             \
828         }
829
830 CSAN_BUS_SET_FUNC(multi, 1, uint8_t)
831 CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
832 CSAN_BUS_SET_FUNC(region, 1, uint8_t)
833 CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
834
835 CSAN_BUS_SET_FUNC(multi, 2, uint16_t)
836 CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
837 CSAN_BUS_SET_FUNC(region, 2, uint16_t)
838 CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
839
840 CSAN_BUS_SET_FUNC(multi, 4, uint32_t)
841 CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
842 CSAN_BUS_SET_FUNC(region, 4, uint32_t)
843 CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
844
845 #if !defined(__amd64__)
846 CSAN_BUS_SET_FUNC(multi, 8, uint64_t)
847 CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t)
848 CSAN_BUS_SET_FUNC(region, 8, uint64_t)
849 CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t)
850 #endif
851