sys/sys/_cscan_atomic.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2019 Andrew Turner
   5  *
   6  * This software was developed by SRI International and the University of
   7  * Cambridge Computer Laboratory (Department of Computer Science and
   8  * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
   9  * DARPA SSITH research programme.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  *
  32  * $FreeBSD$
  33  */
  34
  35 #ifndef _SYS__CSAN_ATOMIC_H_
  36 #define _SYS__CSAN_ATOMIC_H_
  37
  38 #ifndef _MACHINE_ATOMIC_H_
  39 #error do not include this header, use machine/atomic.h
  40 #endif
  41
  42 #define KCSAN_ATOMIC_FUNC_1(op, name, type)                             \
  43         void kcsan_atomic_##op##_##name(volatile type *, type);         \
  44         void kcsan_atomic_##op##_acq_##name(volatile type *, type);     \
  45         void kcsan_atomic_##op##_rel_##name(volatile type *, type)
  46
  47 #define KCSAN_ATOMIC_CMPSET(name, type)                                 \
  48         int kcsan_atomic_cmpset_##name(volatile type *, type, type);    \
  49         int kcsan_atomic_cmpset_acq_##name(volatile type *, type, type); \
  50         int kcsan_atomic_cmpset_rel_##name(volatile type *, type, type)
  51
  52 #define KCSAN_ATOMIC_FCMPSET(name, type)                                \
  53         int kcsan_atomic_fcmpset_##name(volatile type *, type *, type); \
  54         int kcsan_atomic_fcmpset_acq_##name(volatile type *, type *, type); \
  55         int kcsan_atomic_fcmpset_rel_##name(volatile type *, type *, type)
  56
  57 #define KCSAN_ATOMIC_READ(op, name, type)                               \
  58         type kcsan_atomic_##op##_##name(volatile type *, type)
  59
  60 #define KCSAN_ATOMIC_READANDCLEAR(name, type)                           \
  61         type kcsan_atomic_readandclear_##name(volatile type *)
  62
  63 #define KCSAN_ATOMIC_LOAD(name, type)                                   \
  64         type kcsan_atomic_load_##name(volatile type *);                 \
  65         type kcsan_atomic_load_acq_##name(volatile type *)
  66
  67 #define KCSAN_ATOMIC_STORE(name, type)                                  \
  68         void kcsan_atomic_store_##name(volatile type *, type);          \
  69         void kcsan_atomic_store_rel_##name(volatile type *, type)
  70
  71 #define KCSAN_ATOMIC_TEST(op, name, type)                               \
  72         int kcsan_atomic_##op##_##name(volatile type *, u_int)
  73
  74 #define KCSAN_ATOMIC_FUNCS(name, type)                                  \
  75         KCSAN_ATOMIC_FUNC_1(add, name, type);                           \
  76         KCSAN_ATOMIC_FUNC_1(clear, name, type);                         \
  77         KCSAN_ATOMIC_CMPSET(name, type);                                \
  78         KCSAN_ATOMIC_FCMPSET(name, type);                               \
  79         KCSAN_ATOMIC_READ(fetchadd, name, type);                        \
  80         KCSAN_ATOMIC_LOAD(name, type);                                  \
  81         KCSAN_ATOMIC_READANDCLEAR(name, type);                          \
  82         KCSAN_ATOMIC_FUNC_1(set, name, type);                           \
  83         KCSAN_ATOMIC_FUNC_1(subtract, name, type);                      \
  84         KCSAN_ATOMIC_STORE(name, type);                                 \
  85         KCSAN_ATOMIC_READ(swap, name, type);                            \
  86         KCSAN_ATOMIC_TEST(testandclear, name, type);                    \
  87         KCSAN_ATOMIC_TEST(testandset, name, type)
  88
  89 KCSAN_ATOMIC_FUNCS(int, u_int);
  90 KCSAN_ATOMIC_FUNCS(long, u_long);
  91 KCSAN_ATOMIC_FUNCS(ptr, uintptr_t);
  92 KCSAN_ATOMIC_FUNCS(8, uint8_t);
  93 KCSAN_ATOMIC_FUNCS(16, uint16_t);
  94 KCSAN_ATOMIC_FUNCS(32, uint32_t);
  95 KCSAN_ATOMIC_FUNCS(64, uint64_t);
  96
  97 void    kcsan_atomic_thread_fence_acq(void);
  98 void    kcsan_atomic_thread_fence_acq_rel(void);
  99 void    kcsan_atomic_thread_fence_rel(void);
 100 void    kcsan_atomic_thread_fence_seq_cst(void);
 101
 102 #ifndef KCSAN_RUNTIME
 103
 104 #define atomic_add_int                  kcsan_atomic_add_int
 105 #define atomic_add_acq_int              kcsan_atomic_add_acq_int
 106 #define atomic_add_rel_int              kcsan_atomic_add_rel_int
 107 #define atomic_clear_int                kcsan_atomic_clear_int
 108 #define atomic_clear_acq_int            kcsan_atomic_clear_acq_int
 109 #define atomic_clear_rel_int            kcsan_atomic_clear_rel_int
 110 #define atomic_cmpset_int               kcsan_atomic_cmpset_int
 111 #define atomic_cmpset_acq_int           kcsan_atomic_cmpset_acq_int
 112 #define atomic_cmpset_rel_int           kcsan_atomic_cmpset_rel_int
 113 #define atomic_fcmpset_int              kcsan_atomic_fcmpset_int
 114 #define atomic_fcmpset_acq_int          kcsan_atomic_fcmpset_acq_int
 115 #define atomic_fcmpset_rel_int          kcsan_atomic_fcmpset_rel_int
 116 #define atomic_fetchadd_int             kcsan_atomic_fetchadd_int
 117 #define atomic_load_int                 kcsan_atomic_load_int
 118 #define atomic_load_acq_int             kcsan_atomic_load_acq_int
 119 #define atomic_readandclear_int         kcsan_atomic_readandclear_int
 120 #define atomic_set_int                  kcsan_atomic_set_int
 121 #define atomic_set_acq_int              kcsan_atomic_set_acq_int
 122 #define atomic_set_rel_int              kcsan_atomic_set_rel_int
 123 #define atomic_subtract_int             kcsan_atomic_subtract_int
 124 #define atomic_subtract_acq_int         kcsan_atomic_subtract_acq_int
 125 #define atomic_subtract_rel_int         kcsan_atomic_subtract_rel_int
 126 #define atomic_store_int                kcsan_atomic_store_int
 127 #define atomic_store_rel_int            kcsan_atomic_store_rel_int
 128 #define atomic_swap_int                 kcsan_atomic_swap_int
 129 #define atomic_testandclear_int         kcsan_atomic_testandclear_int
 130 #define atomic_testandset_int           kcsan_atomic_testandset_int
 131
 132 #define atomic_add_long                 kcsan_atomic_add_long
 133 #define atomic_add_acq_long             kcsan_atomic_add_acq_long
 134 #define atomic_add_rel_long             kcsan_atomic_add_rel_long
 135 #define atomic_clear_long               kcsan_atomic_clear_long
 136 #define atomic_clear_acq_long           kcsan_atomic_clear_acq_long
 137 #define atomic_clear_rel_long           kcsan_atomic_clear_rel_long
 138 #define atomic_cmpset_long              kcsan_atomic_cmpset_long
 139 #define atomic_cmpset_acq_long          kcsan_atomic_cmpset_acq_long
 140 #define atomic_cmpset_rel_long          kcsan_atomic_cmpset_rel_long
 141 #define atomic_fcmpset_long             kcsan_atomic_fcmpset_long
 142 #define atomic_fcmpset_acq_long         kcsan_atomic_fcmpset_acq_long
 143 #define atomic_fcmpset_rel_long         kcsan_atomic_fcmpset_rel_long
 144 #define atomic_fetchadd_long            kcsan_atomic_fetchadd_long
 145 #define atomic_load_long                kcsan_atomic_load_long
 146 #define atomic_load_acq_long            kcsan_atomic_load_acq_long
 147 #define atomic_readandclear_long        kcsan_atomic_readandclear_long
 148 #define atomic_set_long                 kcsan_atomic_set_long
 149 #define atomic_set_acq_long             kcsan_atomic_set_acq_long
 150 #define atomic_set_rel_long             kcsan_atomic_set_rel_long
 151 #define atomic_subtract_long            kcsan_atomic_subtract_long
 152 #define atomic_subtract_acq_long        kcsan_atomic_subtract_acq_long
 153 #define atomic_subtract_rel_long        kcsan_atomic_subtract_rel_long
 154 #define atomic_store_long               kcsan_atomic_store_long
 155 #define atomic_store_rel_long           kcsan_atomic_store_rel_long
 156 #define atomic_swap_long                kcsan_atomic_swap_long
 157 #define atomic_testandclear_long        kcsan_atomic_testandclear_long
 158 #define atomic_testandset_long          kcsan_atomic_testandset_long
 159
 160 #define atomic_add_ptr                  kcsan_atomic_add_ptr
 161 #define atomic_add_acq_ptr              kcsan_atomic_add_acq_ptr
 162 #define atomic_add_rel_ptr              kcsan_atomic_add_rel_ptr
 163 #define atomic_clear_ptr                kcsan_atomic_clear_ptr
 164 #define atomic_clear_acq_ptr            kcsan_atomic_clear_acq_ptr
 165 #define atomic_clear_rel_ptr            kcsan_atomic_clear_rel_ptr
 166 #define atomic_cmpset_ptr               kcsan_atomic_cmpset_ptr
 167 #define atomic_cmpset_acq_ptr           kcsan_atomic_cmpset_acq_ptr
 168 #define atomic_cmpset_rel_ptr           kcsan_atomic_cmpset_rel_ptr
 169 #define atomic_fcmpset_ptr              kcsan_atomic_fcmpset_ptr
 170 #define atomic_fcmpset_acq_ptr          kcsan_atomic_fcmpset_acq_ptr
 171 #define atomic_fcmpset_rel_ptr          kcsan_atomic_fcmpset_rel_ptr
 172 #define atomic_fetchadd_ptr             kcsan_atomic_fetchadd_ptr
 173 #define atomic_load_ptr(x)              ({                                      \
 174         __typeof(*x) __retptr;                                                  \
 175         __retptr = (void *)kcsan_atomic_load_ptr((volatile uintptr_t *)(x));    \
 176         __retptr;                                                               \
 177 })
 178 #define atomic_load_acq_ptr             kcsan_atomic_load_acq_ptr
 179 #define atomic_readandclear_ptr         kcsan_atomic_readandclear_ptr
 180 #define atomic_set_ptr                  kcsan_atomic_set_ptr
 181 #define atomic_set_acq_ptr              kcsan_atomic_set_acq_ptr
 182 #define atomic_set_rel_ptr              kcsan_atomic_set_rel_ptr
 183 #define atomic_subtract_ptr             kcsan_atomic_subtract_ptr
 184 #define atomic_subtract_acq_ptr         kcsan_atomic_subtract_acq_ptr
 185 #define atomic_subtract_rel_ptr         kcsan_atomic_subtract_rel_ptr
 186 #define atomic_store_ptr                kcsan_atomic_store_ptr
 187 #define atomic_store_rel_ptr            kcsan_atomic_store_rel_ptr
 188 #define atomic_swap_ptr                 kcsan_atomic_swap_ptr
 189 #define atomic_testandclear_ptr         kcsan_atomic_testandclear_ptr
 190 #define atomic_testandset_ptr           kcsan_atomic_testandset_ptr
 191
 192 #define atomic_add_8                    kcsan_atomic_add_8
 193 #define atomic_add_acq_8                kcsan_atomic_add_acq_8
 194 #define atomic_add_rel_8                kcsan_atomic_add_rel_8
 195 #define atomic_clear_8                  kcsan_atomic_clear_8
 196 #define atomic_clear_acq_8              kcsan_atomic_clear_acq_8
 197 #define atomic_clear_rel_8              kcsan_atomic_clear_rel_8
 198 #define atomic_cmpset_8                 kcsan_atomic_cmpset_8
 199 #define atomic_cmpset_acq_8             kcsan_atomic_cmpset_acq_8
 200 #define atomic_cmpset_rel_8             kcsan_atomic_cmpset_rel_8
 201 #define atomic_fcmpset_8                kcsan_atomic_fcmpset_8
 202 #define atomic_fcmpset_acq_8            kcsan_atomic_fcmpset_acq_8
 203 #define atomic_fcmpset_rel_8            kcsan_atomic_fcmpset_rel_8
 204 #define atomic_fetchadd_8               kcsan_atomic_fetchadd_8
 205 #define atomic_load_8                   kcsan_atomic_load_8
 206 #define atomic_load_acq_8               kcsan_atomic_load_acq_8
 207 #define atomic_readandclear_8           kcsan_atomic_readandclear_8
 208 #define atomic_set_8                    kcsan_atomic_set_8
 209 #define atomic_set_acq_8                kcsan_atomic_set_acq_8
 210 #define atomic_set_rel_8                kcsan_atomic_set_rel_8
 211 #define atomic_subtract_8               kcsan_atomic_subtract_8
 212 #define atomic_subtract_acq_8           kcsan_atomic_subtract_acq_8
 213 #define atomic_subtract_rel_8           kcsan_atomic_subtract_rel_8
 214 #define atomic_store_8                  kcsan_atomic_store_8
 215 #define atomic_store_rel_8              kcsan_atomic_store_rel_8
 216 #define atomic_swap_8                   kcsan_atomic_swap_8
 217 #define atomic_testandclear_8           kcsan_atomic_testandclear_8
 218 #define atomic_testandset_8             kcsan_atomic_testandset_8
 219
 220 #define atomic_add_16                   kcsan_atomic_add_16
 221 #define atomic_add_acq_16               kcsan_atomic_add_acq_16
 222 #define atomic_add_rel_16               kcsan_atomic_add_rel_16
 223 #define atomic_clear_16                 kcsan_atomic_clear_16
 224 #define atomic_clear_acq_16             kcsan_atomic_clear_acq_16
 225 #define atomic_clear_rel_16             kcsan_atomic_clear_rel_16
 226 #define atomic_cmpset_16                kcsan_atomic_cmpset_16
 227 #define atomic_cmpset_acq_16            kcsan_atomic_cmpset_acq_16
 228 #define atomic_cmpset_rel_16            kcsan_atomic_cmpset_rel_16
 229 #define atomic_fcmpset_16               kcsan_atomic_fcmpset_16
 230 #define atomic_fcmpset_acq_16           kcsan_atomic_fcmpset_acq_16
 231 #define atomic_fcmpset_rel_16           kcsan_atomic_fcmpset_rel_16
 232 #define atomic_fetchadd_16              kcsan_atomic_fetchadd_16
 233 #define atomic_load_16                  kcsan_atomic_load_16
 234 #define atomic_load_acq_16              kcsan_atomic_load_acq_16
 235 #define atomic_readandclear_16          kcsan_atomic_readandclear_16
 236 #define atomic_set_16                   kcsan_atomic_set_16
 237 #define atomic_set_acq_16               kcsan_atomic_set_acq_16
 238 #define atomic_set_rel_16               kcsan_atomic_set_rel_16
 239 #define atomic_subtract_16              kcsan_atomic_subtract_16
 240 #define atomic_subtract_acq_16          kcsan_atomic_subtract_acq_16
 241 #define atomic_subtract_rel_16          kcsan_atomic_subtract_rel_16
 242 #define atomic_store_16                 kcsan_atomic_store_16
 243 #define atomic_store_rel_16             kcsan_atomic_store_rel_16
 244 #define atomic_swap_16                  kcsan_atomic_swap_16
 245 #define atomic_testandclear_16          kcsan_atomic_testandclear_16
 246 #define atomic_testandset_16            kcsan_atomic_testandset_16
 247
 248 #define atomic_add_32                   kcsan_atomic_add_32
 249 #define atomic_add_acq_32               kcsan_atomic_add_acq_32
 250 #define atomic_add_rel_32               kcsan_atomic_add_rel_32
 251 #define atomic_clear_32                 kcsan_atomic_clear_32
 252 #define atomic_clear_acq_32             kcsan_atomic_clear_acq_32
 253 #define atomic_clear_rel_32             kcsan_atomic_clear_rel_32
 254 #define atomic_cmpset_32                kcsan_atomic_cmpset_32
 255 #define atomic_cmpset_acq_32            kcsan_atomic_cmpset_acq_32
 256 #define atomic_cmpset_rel_32            kcsan_atomic_cmpset_rel_32
 257 #define atomic_fcmpset_32               kcsan_atomic_fcmpset_32
 258 #define atomic_fcmpset_acq_32           kcsan_atomic_fcmpset_acq_32
 259 #define atomic_fcmpset_rel_32           kcsan_atomic_fcmpset_rel_32
 260 #define atomic_fetchadd_32              kcsan_atomic_fetchadd_32
 261 #define atomic_load_32                  kcsan_atomic_load_32
 262 #define atomic_load_acq_32              kcsan_atomic_load_acq_32
 263 #define atomic_readandclear_32          kcsan_atomic_readandclear_32
 264 #define atomic_set_32                   kcsan_atomic_set_32
 265 #define atomic_set_acq_32               kcsan_atomic_set_acq_32
 266 #define atomic_set_rel_32               kcsan_atomic_set_rel_32
 267 #define atomic_subtract_32              kcsan_atomic_subtract_32
 268 #define atomic_subtract_acq_32          kcsan_atomic_subtract_acq_32
 269 #define atomic_subtract_rel_32          kcsan_atomic_subtract_rel_32
 270 #define atomic_store_32                 kcsan_atomic_store_32
 271 #define atomic_store_rel_32             kcsan_atomic_store_rel_32
 272 #define atomic_swap_32                  kcsan_atomic_swap_32
 273 #define atomic_testandclear_32          kcsan_atomic_testandclear_32
 274 #define atomic_testandset_32            kcsan_atomic_testandset_32
 275
 276 #define atomic_add_64                   kcsan_atomic_add_64
 277 #define atomic_add_acq_64               kcsan_atomic_add_acq_64
 278 #define atomic_add_rel_64               kcsan_atomic_add_rel_64
 279 #define atomic_clear_64                 kcsan_atomic_clear_64
 280 #define atomic_clear_acq_64             kcsan_atomic_clear_acq_64
 281 #define atomic_clear_rel_64             kcsan_atomic_clear_rel_64
 282 #define atomic_cmpset_64                kcsan_atomic_cmpset_64
 283 #define atomic_cmpset_acq_64            kcsan_atomic_cmpset_acq_64
 284 #define atomic_cmpset_rel_64            kcsan_atomic_cmpset_rel_64
 285 #define atomic_fcmpset_64               kcsan_atomic_fcmpset_64
 286 #define atomic_fcmpset_acq_64           kcsan_atomic_fcmpset_acq_64
 287 #define atomic_fcmpset_rel_64           kcsan_atomic_fcmpset_rel_64
 288 #define atomic_fetchadd_64              kcsan_atomic_fetchadd_64
 289 #define atomic_load_64                  kcsan_atomic_load_64
 290 #define atomic_load_acq_64              kcsan_atomic_load_acq_64
 291 #define atomic_readandclear_64          kcsan_atomic_readandclear_64
 292 #define atomic_set_64                   kcsan_atomic_set_64
 293 #define atomic_set_acq_64               kcsan_atomic_set_acq_64
 294 #define atomic_set_rel_64               kcsan_atomic_set_rel_64
 295 #define atomic_subtract_64              kcsan_atomic_subtract_64
 296 #define atomic_subtract_acq_64          kcsan_atomic_subtract_acq_64
 297 #define atomic_subtract_rel_64          kcsan_atomic_subtract_rel_64
 298 #define atomic_store_64                 kcsan_atomic_store_64
 299 #define atomic_store_rel_64             kcsan_atomic_store_rel_64
 300 #define atomic_swap_64                  kcsan_atomic_swap_64
 301 #define atomic_testandclear_64          kcsan_atomic_testandclear_64
 302 #define atomic_testandset_64            kcsan_atomic_testandset_64
 303
 304 #define atomic_thread_fence_acq         kcsan_atomic_thread_fence_acq
 305 #define atomic_thread_fence_acq_rel     kcsan_atomic_thread_fence_acq_rel
 306 #define atomic_thread_fence_rel         kcsan_atomic_thread_fence_rel
 307 #define atomic_thread_fence_seq_cst     kcsan_atomic_thread_fence_seq_cst
 308
 309 #endif /* !KCSAN_RUNTIME */
 310
 311 #endif /* !_SYS__CSAN_ATOMIC_H_ */