sys/sys/atomic_san.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2019 Andrew Turner
   5  * Copyright (c) 2021 The FreeBSD Foundation
   6  *
   7  * This software was developed by SRI International and the University of
   8  * Cambridge Computer Laboratory (Department of Computer Science and
   9  * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
  10  * DARPA SSITH research programme.
  11  *
  12  * Portions of this software were written by Mark Johnston under sponsorship
  13  * by the FreeBSD Foundation.
  14  *
  15  * Redistribution and use in source and binary forms, with or without
  16  * modification, are permitted provided that the following conditions
  17  * are met:
  18  * 1. Redistributions of source code must retain the above copyright
  19  *    notice, this list of conditions and the following disclaimer.
  20  * 2. Redistributions in binary form must reproduce the above copyright
  21  *    notice, this list of conditions and the following disclaimer in the
  22  *    documentation and/or other materials provided with the distribution.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  34  * SUCH DAMAGE.
  35  */
  36
  37 #ifndef _SYS_ATOMIC_SAN_H_
  38 #define _SYS_ATOMIC_SAN_H_
  39
  40 #ifndef _MACHINE_ATOMIC_H_
  41 #error do not include this header, use machine/atomic.h
  42 #endif
  43
  44 #include <sys/types.h>
  45
  46 #define ATOMIC_SAN_FUNC_1(sp, op, name, type)                           \
  47         void sp##_atomic_##op##_##name(volatile type *, type);          \
  48         void sp##_atomic_##op##_acq_##name(volatile type *, type);      \
  49         void sp##_atomic_##op##_rel_##name(volatile type *, type)
  50
  51 #define ATOMIC_SAN_CMPSET(sp, name, type)                               \
  52         int sp##_atomic_cmpset_##name(volatile type *, type, type);     \
  53         int sp##_atomic_cmpset_acq_##name(volatile type *, type, type); \
  54         int sp##_atomic_cmpset_rel_##name(volatile type *, type, type)
  55
  56 #define ATOMIC_SAN_FCMPSET(sp, name, type)                              \
  57         int sp##_atomic_fcmpset_##name(volatile type *, type *, type);  \
  58         int sp##_atomic_fcmpset_acq_##name(volatile type *, type *, type); \
  59         int sp##_atomic_fcmpset_rel_##name(volatile type *, type *, type)
  60
  61 #define ATOMIC_SAN_READ(sp, op, name, type)                             \
  62         type sp##_atomic_##op##_##name(volatile type *, type)
  63
  64 #define ATOMIC_SAN_READANDCLEAR(sp, name, type)                         \
  65         type sp##_atomic_readandclear_##name(volatile type *)
  66
  67 #define ATOMIC_SAN_LOAD(sp, name, type)                                 \
  68         type sp##_atomic_load_##name(volatile type *)
  69
  70 #define ATOMIC_SAN_LOAD_ACQ(sp, name, type)                             \
  71         type sp##_atomic_load_acq_##name(volatile type *)
  72
  73 #define ATOMIC_SAN_STORE(sp, name, type)                                \
  74         void sp##_atomic_store_##name(volatile type *, type)
  75
  76 #define ATOMIC_SAN_STORE_REL(sp, name, type)                            \
  77         void sp##_atomic_store_rel_##name(volatile type *, type)
  78
  79 #define ATOMIC_SAN_TEST(sp, op, name, type)                             \
  80         int sp##_atomic_##op##_##name(volatile type *, u_int);          \
  81         int sp##_atomic_##op##_acq_##name(volatile type *, u_int)
  82
  83 #define _ATOMIC_SAN_THREAD_FENCE(sp)                                    \
  84         void sp##_atomic_thread_fence_acq(void);                        \
  85         void sp##_atomic_thread_fence_rel(void);                        \
  86         void sp##_atomic_thread_fence_acq_rel(void);                    \
  87         void sp##_atomic_thread_fence_seq_cst(void);                    \
  88         void sp##_atomic_interrupt_fence(void)
  89
  90 #define ATOMIC_SAN_THREAD_FENCE(sp)                                     \
  91         _ATOMIC_SAN_THREAD_FENCE(sp)
  92
  93 #define ATOMIC_SAN_LOAD_STORE(sp, name, type)                           \
  94         ATOMIC_SAN_LOAD(sp, name, type);                                \
  95         ATOMIC_SAN_STORE(sp, name, type)
  96
  97 #define _ATOMIC_SAN_FUNCS(sp, name, type)                               \
  98         ATOMIC_SAN_FUNC_1(sp, add, name, type);                         \
  99         ATOMIC_SAN_FUNC_1(sp, clear, name, type);                       \
 100         ATOMIC_SAN_CMPSET(sp, name, type);                              \
 101         ATOMIC_SAN_FCMPSET(sp, name, type);                             \
 102         ATOMIC_SAN_READ(sp, fetchadd, name, type);                      \
 103         ATOMIC_SAN_LOAD(sp, name, type);                                \
 104         ATOMIC_SAN_LOAD_ACQ(sp, name, type);                            \
 105         ATOMIC_SAN_READANDCLEAR(sp, name, type);                        \
 106         ATOMIC_SAN_FUNC_1(sp, set, name, type);                         \
 107         ATOMIC_SAN_FUNC_1(sp, subtract, name, type);                    \
 108         ATOMIC_SAN_STORE(sp, name, type);                               \
 109         ATOMIC_SAN_STORE_REL(sp, name, type);                           \
 110         ATOMIC_SAN_READ(sp, swap, name, type);                          \
 111         ATOMIC_SAN_TEST(sp, testandclear, name, type);                  \
 112         ATOMIC_SAN_TEST(sp, testandset, name, type)
 113
 114 #define ATOMIC_SAN_FUNCS(name, type)                                    \
 115         _ATOMIC_SAN_FUNCS(SAN_INTERCEPTOR_PREFIX, name, type)
 116
 117 ATOMIC_SAN_FUNCS(char, uint8_t);
 118 ATOMIC_SAN_FUNCS(short, uint16_t);
 119 ATOMIC_SAN_FUNCS(int, u_int);
 120 ATOMIC_SAN_FUNCS(long, u_long);
 121 ATOMIC_SAN_FUNCS(ptr, uintptr_t);
 122 ATOMIC_SAN_FUNCS(8, uint8_t);
 123 ATOMIC_SAN_FUNCS(16, uint16_t);
 124 ATOMIC_SAN_FUNCS(32, uint32_t);
 125 ATOMIC_SAN_FUNCS(64, uint64_t);
 126 ATOMIC_SAN_LOAD_STORE(SAN_INTERCEPTOR_PREFIX, bool, bool);
 127 ATOMIC_SAN_THREAD_FENCE(SAN_INTERCEPTOR_PREFIX);
 128
 129 #ifndef SAN_RUNTIME
 130
 131 /*
 132  * Redirect uses of an atomic(9) function to the sanitizer's interceptor.
 133  * For instance, KASAN callers of atomic_add_char() will be redirected to
 134  * kasan_atomic_add_char().
 135  */
 136 #define ATOMIC_SAN(func)                                                \
 137         __CONCAT(SAN_INTERCEPTOR_PREFIX, __CONCAT(_atomic_, func))
 138
 139 #define atomic_load_bool                ATOMIC_SAN(load_bool)
 140 #define atomic_store_bool               ATOMIC_SAN(store_bool)
 141
 142 #define atomic_add_char                 ATOMIC_SAN(add_char)
 143 #define atomic_add_acq_char             ATOMIC_SAN(add_acq_char)
 144 #define atomic_add_rel_char             ATOMIC_SAN(add_rel_char)
 145 #define atomic_clear_char               ATOMIC_SAN(clear_char)
 146 #define atomic_clear_acq_char           ATOMIC_SAN(clear_acq_char)
 147 #define atomic_clear_rel_char           ATOMIC_SAN(clear_rel_char)
 148 #define atomic_cmpset_char              ATOMIC_SAN(cmpset_char)
 149 #define atomic_cmpset_acq_char          ATOMIC_SAN(cmpset_acq_char)
 150 #define atomic_cmpset_rel_char          ATOMIC_SAN(cmpset_rel_char)
 151 #define atomic_fcmpset_char             ATOMIC_SAN(fcmpset_char)
 152 #define atomic_fcmpset_acq_char         ATOMIC_SAN(fcmpset_acq_char)
 153 #define atomic_fcmpset_rel_char         ATOMIC_SAN(fcmpset_rel_char)
 154 #define atomic_fetchadd_char            ATOMIC_SAN(fetchadd_char)
 155 #define atomic_load_char                ATOMIC_SAN(load_char)
 156 #define atomic_load_acq_char            ATOMIC_SAN(load_acq_char)
 157 #define atomic_readandclear_char        ATOMIC_SAN(readandclear_char)
 158 #define atomic_set_char                 ATOMIC_SAN(set_char)
 159 #define atomic_set_acq_char             ATOMIC_SAN(set_acq_char)
 160 #define atomic_set_rel_char             ATOMIC_SAN(set_rel_char)
 161 #define atomic_subtract_char            ATOMIC_SAN(subtract_char)
 162 #define atomic_subtract_acq_char        ATOMIC_SAN(subtract_acq_char)
 163 #define atomic_subtract_rel_char        ATOMIC_SAN(subtract_rel_char)
 164 #define atomic_store_char               ATOMIC_SAN(store_char)
 165 #define atomic_store_rel_char           ATOMIC_SAN(store_rel_char)
 166 #define atomic_swap_char                ATOMIC_SAN(swap_char)
 167 #define atomic_testandclear_char        ATOMIC_SAN(testandclear_char)
 168 #define atomic_testandset_char          ATOMIC_SAN(testandset_char)
 169
 170 #define atomic_add_short                ATOMIC_SAN(add_short)
 171 #define atomic_add_acq_short            ATOMIC_SAN(add_acq_short)
 172 #define atomic_add_rel_short            ATOMIC_SAN(add_rel_short)
 173 #define atomic_clear_short              ATOMIC_SAN(clear_short)
 174 #define atomic_clear_acq_short          ATOMIC_SAN(clear_acq_short)
 175 #define atomic_clear_rel_short          ATOMIC_SAN(clear_rel_short)
 176 #define atomic_cmpset_short             ATOMIC_SAN(cmpset_short)
 177 #define atomic_cmpset_acq_short         ATOMIC_SAN(cmpset_acq_short)
 178 #define atomic_cmpset_rel_short         ATOMIC_SAN(cmpset_rel_short)
 179 #define atomic_fcmpset_short            ATOMIC_SAN(fcmpset_short)
 180 #define atomic_fcmpset_acq_short        ATOMIC_SAN(fcmpset_acq_short)
 181 #define atomic_fcmpset_rel_short        ATOMIC_SAN(fcmpset_rel_short)
 182 #define atomic_fetchadd_short           ATOMIC_SAN(fetchadd_short)
 183 #define atomic_load_short               ATOMIC_SAN(load_short)
 184 #define atomic_load_acq_short           ATOMIC_SAN(load_acq_short)
 185 #define atomic_readandclear_short       ATOMIC_SAN(readandclear_short)
 186 #define atomic_set_short                ATOMIC_SAN(set_short)
 187 #define atomic_set_acq_short            ATOMIC_SAN(set_acq_short)
 188 #define atomic_set_rel_short            ATOMIC_SAN(set_rel_short)
 189 #define atomic_subtract_short           ATOMIC_SAN(subtract_short)
 190 #define atomic_subtract_acq_short       ATOMIC_SAN(subtract_acq_short)
 191 #define atomic_subtract_rel_short       ATOMIC_SAN(subtract_rel_short)
 192 #define atomic_store_short              ATOMIC_SAN(store_short)
 193 #define atomic_store_rel_short          ATOMIC_SAN(store_rel_short)
 194 #define atomic_swap_short               ATOMIC_SAN(swap_short)
 195 #define atomic_testandclear_short       ATOMIC_SAN(testandclear_short)
 196 #define atomic_testandset_short         ATOMIC_SAN(testandset_short)
 197
 198 #define atomic_add_int                  ATOMIC_SAN(add_int)
 199 #define atomic_add_acq_int              ATOMIC_SAN(add_acq_int)
 200 #define atomic_add_rel_int              ATOMIC_SAN(add_rel_int)
 201 #define atomic_clear_int                ATOMIC_SAN(clear_int)
 202 #define atomic_clear_acq_int            ATOMIC_SAN(clear_acq_int)
 203 #define atomic_clear_rel_int            ATOMIC_SAN(clear_rel_int)
 204 #define atomic_cmpset_int               ATOMIC_SAN(cmpset_int)
 205 #define atomic_cmpset_acq_int           ATOMIC_SAN(cmpset_acq_int)
 206 #define atomic_cmpset_rel_int           ATOMIC_SAN(cmpset_rel_int)
 207 #define atomic_fcmpset_int              ATOMIC_SAN(fcmpset_int)
 208 #define atomic_fcmpset_acq_int          ATOMIC_SAN(fcmpset_acq_int)
 209 #define atomic_fcmpset_rel_int          ATOMIC_SAN(fcmpset_rel_int)
 210 #define atomic_fetchadd_int             ATOMIC_SAN(fetchadd_int)
 211 #define atomic_load_int                 ATOMIC_SAN(load_int)
 212 #define atomic_load_acq_int             ATOMIC_SAN(load_acq_int)
 213 #define atomic_readandclear_int         ATOMIC_SAN(readandclear_int)
 214 #define atomic_set_int                  ATOMIC_SAN(set_int)
 215 #define atomic_set_acq_int              ATOMIC_SAN(set_acq_int)
 216 #define atomic_set_rel_int              ATOMIC_SAN(set_rel_int)
 217 #define atomic_subtract_int             ATOMIC_SAN(subtract_int)
 218 #define atomic_subtract_acq_int         ATOMIC_SAN(subtract_acq_int)
 219 #define atomic_subtract_rel_int         ATOMIC_SAN(subtract_rel_int)
 220 #define atomic_store_int                ATOMIC_SAN(store_int)
 221 #define atomic_store_rel_int            ATOMIC_SAN(store_rel_int)
 222 #define atomic_swap_int                 ATOMIC_SAN(swap_int)
 223 #define atomic_testandclear_int         ATOMIC_SAN(testandclear_int)
 224 #define atomic_testandset_int           ATOMIC_SAN(testandset_int)
 225
 226 #define atomic_add_long                 ATOMIC_SAN(add_long)
 227 #define atomic_add_acq_long             ATOMIC_SAN(add_acq_long)
 228 #define atomic_add_rel_long             ATOMIC_SAN(add_rel_long)
 229 #define atomic_clear_long               ATOMIC_SAN(clear_long)
 230 #define atomic_clear_acq_long           ATOMIC_SAN(clear_acq_long)
 231 #define atomic_clear_rel_long           ATOMIC_SAN(clear_rel_long)
 232 #define atomic_cmpset_long              ATOMIC_SAN(cmpset_long)
 233 #define atomic_cmpset_acq_long          ATOMIC_SAN(cmpset_acq_long)
 234 #define atomic_cmpset_rel_long          ATOMIC_SAN(cmpset_rel_long)
 235 #define atomic_fcmpset_long             ATOMIC_SAN(fcmpset_long)
 236 #define atomic_fcmpset_acq_long         ATOMIC_SAN(fcmpset_acq_long)
 237 #define atomic_fcmpset_rel_long         ATOMIC_SAN(fcmpset_rel_long)
 238 #define atomic_fetchadd_long            ATOMIC_SAN(fetchadd_long)
 239 #define atomic_load_long                ATOMIC_SAN(load_long)
 240 #define atomic_load_acq_long            ATOMIC_SAN(load_acq_long)
 241 #define atomic_readandclear_long        ATOMIC_SAN(readandclear_long)
 242 #define atomic_set_long                 ATOMIC_SAN(set_long)
 243 #define atomic_set_acq_long             ATOMIC_SAN(set_acq_long)
 244 #define atomic_set_rel_long             ATOMIC_SAN(set_rel_long)
 245 #define atomic_subtract_long            ATOMIC_SAN(subtract_long)
 246 #define atomic_subtract_acq_long        ATOMIC_SAN(subtract_acq_long)
 247 #define atomic_subtract_rel_long        ATOMIC_SAN(subtract_rel_long)
 248 #define atomic_store_long               ATOMIC_SAN(store_long)
 249 #define atomic_store_rel_long           ATOMIC_SAN(store_rel_long)
 250 #define atomic_swap_long                ATOMIC_SAN(swap_long)
 251 #define atomic_testandclear_long        ATOMIC_SAN(testandclear_long)
 252 #define atomic_testandset_long          ATOMIC_SAN(testandset_long)
 253 #define atomic_testandset_acq_long      ATOMIC_SAN(testandset_acq_long)
 254
 255 #define atomic_add_ptr                  ATOMIC_SAN(add_ptr)
 256 #define atomic_add_acq_ptr              ATOMIC_SAN(add_acq_ptr)
 257 #define atomic_add_rel_ptr              ATOMIC_SAN(add_rel_ptr)
 258 #define atomic_clear_ptr                ATOMIC_SAN(clear_ptr)
 259 #define atomic_clear_acq_ptr            ATOMIC_SAN(clear_acq_ptr)
 260 #define atomic_clear_rel_ptr            ATOMIC_SAN(clear_rel_ptr)
 261 #define atomic_cmpset_ptr               ATOMIC_SAN(cmpset_ptr)
 262 #define atomic_cmpset_acq_ptr           ATOMIC_SAN(cmpset_acq_ptr)
 263 #define atomic_cmpset_rel_ptr           ATOMIC_SAN(cmpset_rel_ptr)
 264 #define atomic_fcmpset_ptr              ATOMIC_SAN(fcmpset_ptr)
 265 #define atomic_fcmpset_acq_ptr          ATOMIC_SAN(fcmpset_acq_ptr)
 266 #define atomic_fcmpset_rel_ptr          ATOMIC_SAN(fcmpset_rel_ptr)
 267 #define atomic_fetchadd_ptr             ATOMIC_SAN(fetchadd_ptr)
 268 #define atomic_load_ptr(x)                                              \
 269         ((void *)ATOMIC_SAN(load_ptr)(__DECONST(volatile uintptr_t *, (x))))
 270 #define atomic_load_acq_ptr             ATOMIC_SAN(load_acq_ptr)
 271 #define atomic_load_consume_ptr(x)                                      \
 272         ((void *)atomic_load_acq_ptr((volatile uintptr_t *)(x)))
 273 #define atomic_readandclear_ptr         ATOMIC_SAN(readandclear_ptr)
 274 #define atomic_set_ptr                  ATOMIC_SAN(set_ptr)
 275 #define atomic_set_acq_ptr              ATOMIC_SAN(set_acq_ptr)
 276 #define atomic_set_rel_ptr              ATOMIC_SAN(set_rel_ptr)
 277 #define atomic_subtract_ptr             ATOMIC_SAN(subtract_ptr)
 278 #define atomic_subtract_acq_ptr         ATOMIC_SAN(subtract_acq_ptr)
 279 #define atomic_subtract_rel_ptr         ATOMIC_SAN(subtract_rel_ptr)
 280 #define atomic_store_ptr(x, v)          ({                                      \
 281         __typeof(*x) __value = (v);                                             \
 282         ATOMIC_SAN(store_ptr)((volatile uintptr_t *)(x), (uintptr_t)(__value));\
 283 })
 284 #define atomic_store_rel_ptr            ATOMIC_SAN(store_rel_ptr)
 285 #define atomic_swap_ptr                 ATOMIC_SAN(swap_ptr)
 286 #define atomic_testandclear_ptr         ATOMIC_SAN(testandclear_ptr)
 287 #define atomic_testandset_ptr           ATOMIC_SAN(testandset_ptr)
 288
 289 #define atomic_add_8                    ATOMIC_SAN(add_8)
 290 #define atomic_add_acq_8                ATOMIC_SAN(add_acq_8)
 291 #define atomic_add_rel_8                ATOMIC_SAN(add_rel_8)
 292 #define atomic_clear_8                  ATOMIC_SAN(clear_8)
 293 #define atomic_clear_acq_8              ATOMIC_SAN(clear_acq_8)
 294 #define atomic_clear_rel_8              ATOMIC_SAN(clear_rel_8)
 295 #define atomic_cmpset_8                 ATOMIC_SAN(cmpset_8)
 296 #define atomic_cmpset_acq_8             ATOMIC_SAN(cmpset_acq_8)
 297 #define atomic_cmpset_rel_8             ATOMIC_SAN(cmpset_rel_8)
 298 #define atomic_fcmpset_8                ATOMIC_SAN(fcmpset_8)
 299 #define atomic_fcmpset_acq_8            ATOMIC_SAN(fcmpset_acq_8)
 300 #define atomic_fcmpset_rel_8            ATOMIC_SAN(fcmpset_rel_8)
 301 #define atomic_fetchadd_8               ATOMIC_SAN(fetchadd_8)
 302 #define atomic_load_8                   ATOMIC_SAN(load_8)
 303 #define atomic_load_acq_8               ATOMIC_SAN(load_acq_8)
 304 #define atomic_readandclear_8           ATOMIC_SAN(readandclear_8)
 305 #define atomic_set_8                    ATOMIC_SAN(set_8)
 306 #define atomic_set_acq_8                ATOMIC_SAN(set_acq_8)
 307 #define atomic_set_rel_8                ATOMIC_SAN(set_rel_8)
 308 #define atomic_subtract_8               ATOMIC_SAN(subtract_8)
 309 #define atomic_subtract_acq_8           ATOMIC_SAN(subtract_acq_8)
 310 #define atomic_subtract_rel_8           ATOMIC_SAN(subtract_rel_8)
 311 #define atomic_store_8                  ATOMIC_SAN(store_8)
 312 #define atomic_store_rel_8              ATOMIC_SAN(store_rel_8)
 313 #define atomic_swap_8                   ATOMIC_SAN(swap_8)
 314 #define atomic_testandclear_8           ATOMIC_SAN(testandclear_8)
 315 #define atomic_testandset_8             ATOMIC_SAN(testandset_8)
 316
 317 #define atomic_add_16                   ATOMIC_SAN(add_16)
 318 #define atomic_add_acq_16               ATOMIC_SAN(add_acq_16)
 319 #define atomic_add_rel_16               ATOMIC_SAN(add_rel_16)
 320 #define atomic_clear_16                 ATOMIC_SAN(clear_16)
 321 #define atomic_clear_acq_16             ATOMIC_SAN(clear_acq_16)
 322 #define atomic_clear_rel_16             ATOMIC_SAN(clear_rel_16)
 323 #define atomic_cmpset_16                ATOMIC_SAN(cmpset_16)
 324 #define atomic_cmpset_acq_16            ATOMIC_SAN(cmpset_acq_16)
 325 #define atomic_cmpset_rel_16            ATOMIC_SAN(cmpset_rel_16)
 326 #define atomic_fcmpset_16               ATOMIC_SAN(fcmpset_16)
 327 #define atomic_fcmpset_acq_16           ATOMIC_SAN(fcmpset_acq_16)
 328 #define atomic_fcmpset_rel_16           ATOMIC_SAN(fcmpset_rel_16)
 329 #define atomic_fetchadd_16              ATOMIC_SAN(fetchadd_16)
 330 #define atomic_load_16                  ATOMIC_SAN(load_16)
 331 #define atomic_load_acq_16              ATOMIC_SAN(load_acq_16)
 332 #define atomic_readandclear_16          ATOMIC_SAN(readandclear_16)
 333 #define atomic_set_16                   ATOMIC_SAN(set_16)
 334 #define atomic_set_acq_16               ATOMIC_SAN(set_acq_16)
 335 #define atomic_set_rel_16               ATOMIC_SAN(set_rel_16)
 336 #define atomic_subtract_16              ATOMIC_SAN(subtract_16)
 337 #define atomic_subtract_acq_16          ATOMIC_SAN(subtract_acq_16)
 338 #define atomic_subtract_rel_16          ATOMIC_SAN(subtract_rel_16)
 339 #define atomic_store_16                 ATOMIC_SAN(store_16)
 340 #define atomic_store_rel_16             ATOMIC_SAN(store_rel_16)
 341 #define atomic_swap_16                  ATOMIC_SAN(swap_16)
 342 #define atomic_testandclear_16          ATOMIC_SAN(testandclear_16)
 343 #define atomic_testandset_16            ATOMIC_SAN(testandset_16)
 344
 345 #define atomic_add_32                   ATOMIC_SAN(add_32)
 346 #define atomic_add_acq_32               ATOMIC_SAN(add_acq_32)
 347 #define atomic_add_rel_32               ATOMIC_SAN(add_rel_32)
 348 #define atomic_clear_32                 ATOMIC_SAN(clear_32)
 349 #define atomic_clear_acq_32             ATOMIC_SAN(clear_acq_32)
 350 #define atomic_clear_rel_32             ATOMIC_SAN(clear_rel_32)
 351 #define atomic_cmpset_32                ATOMIC_SAN(cmpset_32)
 352 #define atomic_cmpset_acq_32            ATOMIC_SAN(cmpset_acq_32)
 353 #define atomic_cmpset_rel_32            ATOMIC_SAN(cmpset_rel_32)
 354 #define atomic_fcmpset_32               ATOMIC_SAN(fcmpset_32)
 355 #define atomic_fcmpset_acq_32           ATOMIC_SAN(fcmpset_acq_32)
 356 #define atomic_fcmpset_rel_32           ATOMIC_SAN(fcmpset_rel_32)
 357 #define atomic_fetchadd_32              ATOMIC_SAN(fetchadd_32)
 358 #define atomic_load_32                  ATOMIC_SAN(load_32)
 359 #define atomic_load_acq_32              ATOMIC_SAN(load_acq_32)
 360 #define atomic_readandclear_32          ATOMIC_SAN(readandclear_32)
 361 #define atomic_set_32                   ATOMIC_SAN(set_32)
 362 #define atomic_set_acq_32               ATOMIC_SAN(set_acq_32)
 363 #define atomic_set_rel_32               ATOMIC_SAN(set_rel_32)
 364 #define atomic_subtract_32              ATOMIC_SAN(subtract_32)
 365 #define atomic_subtract_acq_32          ATOMIC_SAN(subtract_acq_32)
 366 #define atomic_subtract_rel_32          ATOMIC_SAN(subtract_rel_32)
 367 #define atomic_store_32                 ATOMIC_SAN(store_32)
 368 #define atomic_store_rel_32             ATOMIC_SAN(store_rel_32)
 369 #define atomic_swap_32                  ATOMIC_SAN(swap_32)
 370 #define atomic_testandclear_32          ATOMIC_SAN(testandclear_32)
 371 #define atomic_testandset_32            ATOMIC_SAN(testandset_32)
 372
 373 #define atomic_add_64                   ATOMIC_SAN(add_64)
 374 #define atomic_add_acq_64               ATOMIC_SAN(add_acq_64)
 375 #define atomic_add_rel_64               ATOMIC_SAN(add_rel_64)
 376 #define atomic_clear_64                 ATOMIC_SAN(clear_64)
 377 #define atomic_clear_acq_64             ATOMIC_SAN(clear_acq_64)
 378 #define atomic_clear_rel_64             ATOMIC_SAN(clear_rel_64)
 379 #define atomic_cmpset_64                ATOMIC_SAN(cmpset_64)
 380 #define atomic_cmpset_acq_64            ATOMIC_SAN(cmpset_acq_64)
 381 #define atomic_cmpset_rel_64            ATOMIC_SAN(cmpset_rel_64)
 382 #define atomic_fcmpset_64               ATOMIC_SAN(fcmpset_64)
 383 #define atomic_fcmpset_acq_64           ATOMIC_SAN(fcmpset_acq_64)
 384 #define atomic_fcmpset_rel_64           ATOMIC_SAN(fcmpset_rel_64)
 385 #define atomic_fetchadd_64              ATOMIC_SAN(fetchadd_64)
 386 #define atomic_load_64                  ATOMIC_SAN(load_64)
 387 #define atomic_load_acq_64              ATOMIC_SAN(load_acq_64)
 388 #define atomic_readandclear_64          ATOMIC_SAN(readandclear_64)
 389 #define atomic_set_64                   ATOMIC_SAN(set_64)
 390 #define atomic_set_acq_64               ATOMIC_SAN(set_acq_64)
 391 #define atomic_set_rel_64               ATOMIC_SAN(set_rel_64)
 392 #define atomic_subtract_64              ATOMIC_SAN(subtract_64)
 393 #define atomic_subtract_acq_64          ATOMIC_SAN(subtract_acq_64)
 394 #define atomic_subtract_rel_64          ATOMIC_SAN(subtract_rel_64)
 395 #define atomic_store_64                 ATOMIC_SAN(store_64)
 396 #define atomic_store_rel_64             ATOMIC_SAN(store_rel_64)
 397 #define atomic_swap_64                  ATOMIC_SAN(swap_64)
 398 #define atomic_testandclear_64          ATOMIC_SAN(testandclear_64)
 399 #define atomic_testandset_64            ATOMIC_SAN(testandset_64)
 400
 401 #define atomic_thread_fence_acq         ATOMIC_SAN(thread_fence_acq)
 402 #define atomic_thread_fence_acq_rel     ATOMIC_SAN(thread_fence_acq_rel)
 403 #define atomic_thread_fence_rel         ATOMIC_SAN(thread_fence_rel)
 404 #define atomic_thread_fence_seq_cst     ATOMIC_SAN(thread_fence_seq_cst)
 405 #define atomic_interrupt_fence          ATOMIC_SAN(interrupt_fence)
 406
 407 #endif /* !SAN_RUNTIME */
 408
 409 #endif /* !_SYS_ATOMIC_SAN_H_ */