1 /*===---- __clang_cuda_device_functions.h - CUDA runtime support -----------===
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 *===-----------------------------------------------------------------------===
10 #ifndef __CLANG_CUDA_DEVICE_FUNCTIONS_H__
11 #define __CLANG_CUDA_DEVICE_FUNCTIONS_H__
14 #if CUDA_VERSION < 9000
15 #error This file is intended to be used with CUDA-9+ only.
19 // __DEVICE__ is a helper macro with common set of attributes for the wrappers
20 // we implement in this file. We need static in order to avoid emitting unused
21 // functions and __forceinline__ helps inlining these wrappers at -O1.
22 #pragma push_macro("__DEVICE__")
24 #define __DEVICE__ static __attribute__((always_inline))
26 #define __DEVICE__ static __device__ __forceinline__
29 // libdevice provides fast low precision and slow full-recision implementations
30 // for some functions. Which one gets selected depends on
31 // __CLANG_CUDA_APPROX_TRANSCENDENTALS__ which gets defined by clang if
32 // -ffast-math or -fcuda-approx-transcendentals are in effect.
33 #pragma push_macro("__FAST_OR_SLOW")
34 #if defined(__CLANG_CUDA_APPROX_TRANSCENDENTALS__)
35 #define __FAST_OR_SLOW(fast, slow) fast
37 #define __FAST_OR_SLOW(fast, slow) slow
40 // For C++ 17 we need to include noexcept attribute to be compatible
41 // with the header-defined version. This may be removed once
42 // variant is supported.
43 #if defined(_OPENMP) && defined(__cplusplus) && __cplusplus >= 201703L
44 #define __NOEXCEPT noexcept
49 __DEVICE__ int __all(int __a) { return __nvvm_vote_all(__a); }
50 __DEVICE__ int __any(int __a) { return __nvvm_vote_any(__a); }
51 __DEVICE__ unsigned int __ballot(int __a) { return __nvvm_vote_ballot(__a); }
52 __DEVICE__ unsigned int __brev(unsigned int __a) { return __nv_brev(__a); }
53 __DEVICE__ unsigned long long __brevll(unsigned long long __a) {
54 return __nv_brevll(__a);
56 #if defined(__cplusplus)
57 __DEVICE__ void __brkpt() { asm volatile("brkpt;"); }
58 __DEVICE__ void __brkpt(int __a) { __brkpt(); }
60 __DEVICE__ void __attribute__((overloadable)) __brkpt(void) { asm volatile("brkpt;"); }
61 __DEVICE__ void __attribute__((overloadable)) __brkpt(int __a) { __brkpt(); }
63 __DEVICE__ unsigned int __byte_perm(unsigned int __a, unsigned int __b,
65 return __nv_byte_perm(__a, __b, __c);
67 __DEVICE__ int __clz(int __a) { return __nv_clz(__a); }
68 __DEVICE__ int __clzll(long long __a) { return __nv_clzll(__a); }
69 __DEVICE__ float __cosf(float __a) { return __nv_fast_cosf(__a); }
70 __DEVICE__ double __dAtomicAdd(double *__p, double __v) {
71 return __nvvm_atom_add_gen_d(__p, __v);
73 __DEVICE__ double __dAtomicAdd_block(double *__p, double __v) {
74 return __nvvm_atom_cta_add_gen_d(__p, __v);
76 __DEVICE__ double __dAtomicAdd_system(double *__p, double __v) {
77 return __nvvm_atom_sys_add_gen_d(__p, __v);
79 __DEVICE__ double __dadd_rd(double __a, double __b) {
80 return __nv_dadd_rd(__a, __b);
82 __DEVICE__ double __dadd_rn(double __a, double __b) {
83 return __nv_dadd_rn(__a, __b);
85 __DEVICE__ double __dadd_ru(double __a, double __b) {
86 return __nv_dadd_ru(__a, __b);
88 __DEVICE__ double __dadd_rz(double __a, double __b) {
89 return __nv_dadd_rz(__a, __b);
91 __DEVICE__ double __ddiv_rd(double __a, double __b) {
92 return __nv_ddiv_rd(__a, __b);
94 __DEVICE__ double __ddiv_rn(double __a, double __b) {
95 return __nv_ddiv_rn(__a, __b);
97 __DEVICE__ double __ddiv_ru(double __a, double __b) {
98 return __nv_ddiv_ru(__a, __b);
100 __DEVICE__ double __ddiv_rz(double __a, double __b) {
101 return __nv_ddiv_rz(__a, __b);
103 __DEVICE__ double __dmul_rd(double __a, double __b) {
104 return __nv_dmul_rd(__a, __b);
106 __DEVICE__ double __dmul_rn(double __a, double __b) {
107 return __nv_dmul_rn(__a, __b);
109 __DEVICE__ double __dmul_ru(double __a, double __b) {
110 return __nv_dmul_ru(__a, __b);
112 __DEVICE__ double __dmul_rz(double __a, double __b) {
113 return __nv_dmul_rz(__a, __b);
115 __DEVICE__ float __double2float_rd(double __a) {
116 return __nv_double2float_rd(__a);
118 __DEVICE__ float __double2float_rn(double __a) {
119 return __nv_double2float_rn(__a);
121 __DEVICE__ float __double2float_ru(double __a) {
122 return __nv_double2float_ru(__a);
124 __DEVICE__ float __double2float_rz(double __a) {
125 return __nv_double2float_rz(__a);
127 __DEVICE__ int __double2hiint(double __a) { return __nv_double2hiint(__a); }
128 __DEVICE__ int __double2int_rd(double __a) { return __nv_double2int_rd(__a); }
129 __DEVICE__ int __double2int_rn(double __a) { return __nv_double2int_rn(__a); }
130 __DEVICE__ int __double2int_ru(double __a) { return __nv_double2int_ru(__a); }
131 __DEVICE__ int __double2int_rz(double __a) { return __nv_double2int_rz(__a); }
132 __DEVICE__ long long __double2ll_rd(double __a) {
133 return __nv_double2ll_rd(__a);
135 __DEVICE__ long long __double2ll_rn(double __a) {
136 return __nv_double2ll_rn(__a);
138 __DEVICE__ long long __double2ll_ru(double __a) {
139 return __nv_double2ll_ru(__a);
141 __DEVICE__ long long __double2ll_rz(double __a) {
142 return __nv_double2ll_rz(__a);
144 __DEVICE__ int __double2loint(double __a) { return __nv_double2loint(__a); }
145 __DEVICE__ unsigned int __double2uint_rd(double __a) {
146 return __nv_double2uint_rd(__a);
148 __DEVICE__ unsigned int __double2uint_rn(double __a) {
149 return __nv_double2uint_rn(__a);
151 __DEVICE__ unsigned int __double2uint_ru(double __a) {
152 return __nv_double2uint_ru(__a);
154 __DEVICE__ unsigned int __double2uint_rz(double __a) {
155 return __nv_double2uint_rz(__a);
157 __DEVICE__ unsigned long long __double2ull_rd(double __a) {
158 return __nv_double2ull_rd(__a);
160 __DEVICE__ unsigned long long __double2ull_rn(double __a) {
161 return __nv_double2ull_rn(__a);
163 __DEVICE__ unsigned long long __double2ull_ru(double __a) {
164 return __nv_double2ull_ru(__a);
166 __DEVICE__ unsigned long long __double2ull_rz(double __a) {
167 return __nv_double2ull_rz(__a);
169 __DEVICE__ long long __double_as_longlong(double __a) {
170 return __nv_double_as_longlong(__a);
172 __DEVICE__ double __drcp_rd(double __a) { return __nv_drcp_rd(__a); }
173 __DEVICE__ double __drcp_rn(double __a) { return __nv_drcp_rn(__a); }
174 __DEVICE__ double __drcp_ru(double __a) { return __nv_drcp_ru(__a); }
175 __DEVICE__ double __drcp_rz(double __a) { return __nv_drcp_rz(__a); }
176 __DEVICE__ double __dsqrt_rd(double __a) { return __nv_dsqrt_rd(__a); }
177 __DEVICE__ double __dsqrt_rn(double __a) { return __nv_dsqrt_rn(__a); }
178 __DEVICE__ double __dsqrt_ru(double __a) { return __nv_dsqrt_ru(__a); }
179 __DEVICE__ double __dsqrt_rz(double __a) { return __nv_dsqrt_rz(__a); }
180 __DEVICE__ double __dsub_rd(double __a, double __b) {
181 return __nv_dsub_rd(__a, __b);
183 __DEVICE__ double __dsub_rn(double __a, double __b) {
184 return __nv_dsub_rn(__a, __b);
186 __DEVICE__ double __dsub_ru(double __a, double __b) {
187 return __nv_dsub_ru(__a, __b);
189 __DEVICE__ double __dsub_rz(double __a, double __b) {
190 return __nv_dsub_rz(__a, __b);
192 __DEVICE__ float __exp10f(float __a) { return __nv_fast_exp10f(__a); }
193 __DEVICE__ float __expf(float __a) { return __nv_fast_expf(__a); }
194 __DEVICE__ float __fAtomicAdd(float *__p, float __v) {
195 return __nvvm_atom_add_gen_f(__p, __v);
197 __DEVICE__ float __fAtomicAdd_block(float *__p, float __v) {
198 return __nvvm_atom_cta_add_gen_f(__p, __v);
200 __DEVICE__ float __fAtomicAdd_system(float *__p, float __v) {
201 return __nvvm_atom_sys_add_gen_f(__p, __v);
203 __DEVICE__ float __fAtomicExch(float *__p, float __v) {
204 return __nv_int_as_float(
205 __nvvm_atom_xchg_gen_i((int *)__p, __nv_float_as_int(__v)));
207 __DEVICE__ float __fAtomicExch_block(float *__p, float __v) {
208 return __nv_int_as_float(
209 __nvvm_atom_cta_xchg_gen_i((int *)__p, __nv_float_as_int(__v)));
211 __DEVICE__ float __fAtomicExch_system(float *__p, float __v) {
212 return __nv_int_as_float(
213 __nvvm_atom_sys_xchg_gen_i((int *)__p, __nv_float_as_int(__v)));
215 __DEVICE__ float __fadd_rd(float __a, float __b) {
216 return __nv_fadd_rd(__a, __b);
218 __DEVICE__ float __fadd_rn(float __a, float __b) {
219 return __nv_fadd_rn(__a, __b);
221 __DEVICE__ float __fadd_ru(float __a, float __b) {
222 return __nv_fadd_ru(__a, __b);
224 __DEVICE__ float __fadd_rz(float __a, float __b) {
225 return __nv_fadd_rz(__a, __b);
227 __DEVICE__ float __fdiv_rd(float __a, float __b) {
228 return __nv_fdiv_rd(__a, __b);
230 __DEVICE__ float __fdiv_rn(float __a, float __b) {
231 return __nv_fdiv_rn(__a, __b);
233 __DEVICE__ float __fdiv_ru(float __a, float __b) {
234 return __nv_fdiv_ru(__a, __b);
236 __DEVICE__ float __fdiv_rz(float __a, float __b) {
237 return __nv_fdiv_rz(__a, __b);
239 __DEVICE__ float __fdividef(float __a, float __b) {
240 return __nv_fast_fdividef(__a, __b);
242 __DEVICE__ int __ffs(int __a) { return __nv_ffs(__a); }
243 __DEVICE__ int __ffsll(long long __a) { return __nv_ffsll(__a); }
244 __DEVICE__ int __finite(double __a) { return __nv_isfinited(__a); }
245 __DEVICE__ int __finitef(float __a) { return __nv_finitef(__a); }
247 __DEVICE__ int __finitel(long double __a);
249 __DEVICE__ int __float2int_rd(float __a) { return __nv_float2int_rd(__a); }
250 __DEVICE__ int __float2int_rn(float __a) { return __nv_float2int_rn(__a); }
251 __DEVICE__ int __float2int_ru(float __a) { return __nv_float2int_ru(__a); }
252 __DEVICE__ int __float2int_rz(float __a) { return __nv_float2int_rz(__a); }
253 __DEVICE__ long long __float2ll_rd(float __a) { return __nv_float2ll_rd(__a); }
254 __DEVICE__ long long __float2ll_rn(float __a) { return __nv_float2ll_rn(__a); }
255 __DEVICE__ long long __float2ll_ru(float __a) { return __nv_float2ll_ru(__a); }
256 __DEVICE__ long long __float2ll_rz(float __a) { return __nv_float2ll_rz(__a); }
257 __DEVICE__ unsigned int __float2uint_rd(float __a) {
258 return __nv_float2uint_rd(__a);
260 __DEVICE__ unsigned int __float2uint_rn(float __a) {
261 return __nv_float2uint_rn(__a);
263 __DEVICE__ unsigned int __float2uint_ru(float __a) {
264 return __nv_float2uint_ru(__a);
266 __DEVICE__ unsigned int __float2uint_rz(float __a) {
267 return __nv_float2uint_rz(__a);
269 __DEVICE__ unsigned long long __float2ull_rd(float __a) {
270 return __nv_float2ull_rd(__a);
272 __DEVICE__ unsigned long long __float2ull_rn(float __a) {
273 return __nv_float2ull_rn(__a);
275 __DEVICE__ unsigned long long __float2ull_ru(float __a) {
276 return __nv_float2ull_ru(__a);
278 __DEVICE__ unsigned long long __float2ull_rz(float __a) {
279 return __nv_float2ull_rz(__a);
281 __DEVICE__ int __float_as_int(float __a) { return __nv_float_as_int(__a); }
282 __DEVICE__ unsigned int __float_as_uint(float __a) {
283 return __nv_float_as_uint(__a);
285 __DEVICE__ double __fma_rd(double __a, double __b, double __c) {
286 return __nv_fma_rd(__a, __b, __c);
288 __DEVICE__ double __fma_rn(double __a, double __b, double __c) {
289 return __nv_fma_rn(__a, __b, __c);
291 __DEVICE__ double __fma_ru(double __a, double __b, double __c) {
292 return __nv_fma_ru(__a, __b, __c);
294 __DEVICE__ double __fma_rz(double __a, double __b, double __c) {
295 return __nv_fma_rz(__a, __b, __c);
297 __DEVICE__ float __fmaf_ieee_rd(float __a, float __b, float __c) {
298 return __nv_fmaf_ieee_rd(__a, __b, __c);
300 __DEVICE__ float __fmaf_ieee_rn(float __a, float __b, float __c) {
301 return __nv_fmaf_ieee_rn(__a, __b, __c);
303 __DEVICE__ float __fmaf_ieee_ru(float __a, float __b, float __c) {
304 return __nv_fmaf_ieee_ru(__a, __b, __c);
306 __DEVICE__ float __fmaf_ieee_rz(float __a, float __b, float __c) {
307 return __nv_fmaf_ieee_rz(__a, __b, __c);
309 __DEVICE__ float __fmaf_rd(float __a, float __b, float __c) {
310 return __nv_fmaf_rd(__a, __b, __c);
312 __DEVICE__ float __fmaf_rn(float __a, float __b, float __c) {
313 return __nv_fmaf_rn(__a, __b, __c);
315 __DEVICE__ float __fmaf_ru(float __a, float __b, float __c) {
316 return __nv_fmaf_ru(__a, __b, __c);
318 __DEVICE__ float __fmaf_rz(float __a, float __b, float __c) {
319 return __nv_fmaf_rz(__a, __b, __c);
321 __DEVICE__ float __fmul_rd(float __a, float __b) {
322 return __nv_fmul_rd(__a, __b);
324 __DEVICE__ float __fmul_rn(float __a, float __b) {
325 return __nv_fmul_rn(__a, __b);
327 __DEVICE__ float __fmul_ru(float __a, float __b) {
328 return __nv_fmul_ru(__a, __b);
330 __DEVICE__ float __fmul_rz(float __a, float __b) {
331 return __nv_fmul_rz(__a, __b);
333 __DEVICE__ float __frcp_rd(float __a) { return __nv_frcp_rd(__a); }
334 __DEVICE__ float __frcp_rn(float __a) { return __nv_frcp_rn(__a); }
335 __DEVICE__ float __frcp_ru(float __a) { return __nv_frcp_ru(__a); }
336 __DEVICE__ float __frcp_rz(float __a) { return __nv_frcp_rz(__a); }
337 __DEVICE__ float __frsqrt_rn(float __a) { return __nv_frsqrt_rn(__a); }
338 __DEVICE__ float __fsqrt_rd(float __a) { return __nv_fsqrt_rd(__a); }
339 __DEVICE__ float __fsqrt_rn(float __a) { return __nv_fsqrt_rn(__a); }
340 __DEVICE__ float __fsqrt_ru(float __a) { return __nv_fsqrt_ru(__a); }
341 __DEVICE__ float __fsqrt_rz(float __a) { return __nv_fsqrt_rz(__a); }
342 __DEVICE__ float __fsub_rd(float __a, float __b) {
343 return __nv_fsub_rd(__a, __b);
345 __DEVICE__ float __fsub_rn(float __a, float __b) {
346 return __nv_fsub_rn(__a, __b);
348 __DEVICE__ float __fsub_ru(float __a, float __b) {
349 return __nv_fsub_ru(__a, __b);
351 __DEVICE__ float __fsub_rz(float __a, float __b) {
352 return __nv_fsub_rz(__a, __b);
354 __DEVICE__ int __hadd(int __a, int __b) { return __nv_hadd(__a, __b); }
355 __DEVICE__ double __hiloint2double(int __a, int __b) {
356 return __nv_hiloint2double(__a, __b);
358 __DEVICE__ int __iAtomicAdd(int *__p, int __v) {
359 return __nvvm_atom_add_gen_i(__p, __v);
361 __DEVICE__ int __iAtomicAdd_block(int *__p, int __v) {
362 __nvvm_atom_cta_add_gen_i(__p, __v);
364 __DEVICE__ int __iAtomicAdd_system(int *__p, int __v) {
365 __nvvm_atom_sys_add_gen_i(__p, __v);
367 __DEVICE__ int __iAtomicAnd(int *__p, int __v) {
368 return __nvvm_atom_and_gen_i(__p, __v);
370 __DEVICE__ int __iAtomicAnd_block(int *__p, int __v) {
371 return __nvvm_atom_cta_and_gen_i(__p, __v);
373 __DEVICE__ int __iAtomicAnd_system(int *__p, int __v) {
374 return __nvvm_atom_sys_and_gen_i(__p, __v);
376 __DEVICE__ int __iAtomicCAS(int *__p, int __cmp, int __v) {
377 return __nvvm_atom_cas_gen_i(__p, __cmp, __v);
379 __DEVICE__ int __iAtomicCAS_block(int *__p, int __cmp, int __v) {
380 return __nvvm_atom_cta_cas_gen_i(__p, __cmp, __v);
382 __DEVICE__ int __iAtomicCAS_system(int *__p, int __cmp, int __v) {
383 return __nvvm_atom_sys_cas_gen_i(__p, __cmp, __v);
385 __DEVICE__ int __iAtomicExch(int *__p, int __v) {
386 return __nvvm_atom_xchg_gen_i(__p, __v);
388 __DEVICE__ int __iAtomicExch_block(int *__p, int __v) {
389 return __nvvm_atom_cta_xchg_gen_i(__p, __v);
391 __DEVICE__ int __iAtomicExch_system(int *__p, int __v) {
392 return __nvvm_atom_sys_xchg_gen_i(__p, __v);
394 __DEVICE__ int __iAtomicMax(int *__p, int __v) {
395 return __nvvm_atom_max_gen_i(__p, __v);
397 __DEVICE__ int __iAtomicMax_block(int *__p, int __v) {
398 return __nvvm_atom_cta_max_gen_i(__p, __v);
400 __DEVICE__ int __iAtomicMax_system(int *__p, int __v) {
401 return __nvvm_atom_sys_max_gen_i(__p, __v);
403 __DEVICE__ int __iAtomicMin(int *__p, int __v) {
404 return __nvvm_atom_min_gen_i(__p, __v);
406 __DEVICE__ int __iAtomicMin_block(int *__p, int __v) {
407 return __nvvm_atom_cta_min_gen_i(__p, __v);
409 __DEVICE__ int __iAtomicMin_system(int *__p, int __v) {
410 return __nvvm_atom_sys_min_gen_i(__p, __v);
412 __DEVICE__ int __iAtomicOr(int *__p, int __v) {
413 return __nvvm_atom_or_gen_i(__p, __v);
415 __DEVICE__ int __iAtomicOr_block(int *__p, int __v) {
416 return __nvvm_atom_cta_or_gen_i(__p, __v);
418 __DEVICE__ int __iAtomicOr_system(int *__p, int __v) {
419 return __nvvm_atom_sys_or_gen_i(__p, __v);
421 __DEVICE__ int __iAtomicXor(int *__p, int __v) {
422 return __nvvm_atom_xor_gen_i(__p, __v);
424 __DEVICE__ int __iAtomicXor_block(int *__p, int __v) {
425 return __nvvm_atom_cta_xor_gen_i(__p, __v);
427 __DEVICE__ int __iAtomicXor_system(int *__p, int __v) {
428 return __nvvm_atom_sys_xor_gen_i(__p, __v);
430 __DEVICE__ long long __illAtomicMax(long long *__p, long long __v) {
431 return __nvvm_atom_max_gen_ll(__p, __v);
433 __DEVICE__ long long __illAtomicMax_block(long long *__p, long long __v) {
434 return __nvvm_atom_cta_max_gen_ll(__p, __v);
436 __DEVICE__ long long __illAtomicMax_system(long long *__p, long long __v) {
437 return __nvvm_atom_sys_max_gen_ll(__p, __v);
439 __DEVICE__ long long __illAtomicMin(long long *__p, long long __v) {
440 return __nvvm_atom_min_gen_ll(__p, __v);
442 __DEVICE__ long long __illAtomicMin_block(long long *__p, long long __v) {
443 return __nvvm_atom_cta_min_gen_ll(__p, __v);
445 __DEVICE__ long long __illAtomicMin_system(long long *__p, long long __v) {
446 return __nvvm_atom_sys_min_gen_ll(__p, __v);
448 __DEVICE__ double __int2double_rn(int __a) { return __nv_int2double_rn(__a); }
449 __DEVICE__ float __int2float_rd(int __a) { return __nv_int2float_rd(__a); }
450 __DEVICE__ float __int2float_rn(int __a) { return __nv_int2float_rn(__a); }
451 __DEVICE__ float __int2float_ru(int __a) { return __nv_int2float_ru(__a); }
452 __DEVICE__ float __int2float_rz(int __a) { return __nv_int2float_rz(__a); }
453 __DEVICE__ float __int_as_float(int __a) { return __nv_int_as_float(__a); }
454 __DEVICE__ int __isfinited(double __a) { return __nv_isfinited(__a); }
455 __DEVICE__ int __isinf(double __a) { return __nv_isinfd(__a); }
456 __DEVICE__ int __isinff(float __a) { return __nv_isinff(__a); }
458 __DEVICE__ int __isinfl(long double __a);
460 __DEVICE__ int __isnan(double __a) { return __nv_isnand(__a); }
461 __DEVICE__ int __isnanf(float __a) { return __nv_isnanf(__a); }
463 __DEVICE__ int __isnanl(long double __a);
465 __DEVICE__ double __ll2double_rd(long long __a) {
466 return __nv_ll2double_rd(__a);
468 __DEVICE__ double __ll2double_rn(long long __a) {
469 return __nv_ll2double_rn(__a);
471 __DEVICE__ double __ll2double_ru(long long __a) {
472 return __nv_ll2double_ru(__a);
474 __DEVICE__ double __ll2double_rz(long long __a) {
475 return __nv_ll2double_rz(__a);
477 __DEVICE__ float __ll2float_rd(long long __a) { return __nv_ll2float_rd(__a); }
478 __DEVICE__ float __ll2float_rn(long long __a) { return __nv_ll2float_rn(__a); }
479 __DEVICE__ float __ll2float_ru(long long __a) { return __nv_ll2float_ru(__a); }
480 __DEVICE__ float __ll2float_rz(long long __a) { return __nv_ll2float_rz(__a); }
481 __DEVICE__ long long __llAtomicAnd(long long *__p, long long __v) {
482 return __nvvm_atom_and_gen_ll(__p, __v);
484 __DEVICE__ long long __llAtomicAnd_block(long long *__p, long long __v) {
485 return __nvvm_atom_cta_and_gen_ll(__p, __v);
487 __DEVICE__ long long __llAtomicAnd_system(long long *__p, long long __v) {
488 return __nvvm_atom_sys_and_gen_ll(__p, __v);
490 __DEVICE__ long long __llAtomicOr(long long *__p, long long __v) {
491 return __nvvm_atom_or_gen_ll(__p, __v);
493 __DEVICE__ long long __llAtomicOr_block(long long *__p, long long __v) {
494 return __nvvm_atom_cta_or_gen_ll(__p, __v);
496 __DEVICE__ long long __llAtomicOr_system(long long *__p, long long __v) {
497 return __nvvm_atom_sys_or_gen_ll(__p, __v);
499 __DEVICE__ long long __llAtomicXor(long long *__p, long long __v) {
500 return __nvvm_atom_xor_gen_ll(__p, __v);
502 __DEVICE__ long long __llAtomicXor_block(long long *__p, long long __v) {
503 return __nvvm_atom_cta_xor_gen_ll(__p, __v);
505 __DEVICE__ long long __llAtomicXor_system(long long *__p, long long __v) {
506 return __nvvm_atom_sys_xor_gen_ll(__p, __v);
508 __DEVICE__ float __log10f(float __a) { return __nv_fast_log10f(__a); }
509 __DEVICE__ float __log2f(float __a) { return __nv_fast_log2f(__a); }
510 __DEVICE__ float __logf(float __a) { return __nv_fast_logf(__a); }
511 __DEVICE__ double __longlong_as_double(long long __a) {
512 return __nv_longlong_as_double(__a);
514 __DEVICE__ int __mul24(int __a, int __b) { return __nv_mul24(__a, __b); }
515 __DEVICE__ long long __mul64hi(long long __a, long long __b) {
516 return __nv_mul64hi(__a, __b);
518 __DEVICE__ int __mulhi(int __a, int __b) { return __nv_mulhi(__a, __b); }
519 __DEVICE__ unsigned int __pm0(void) { return __nvvm_read_ptx_sreg_pm0(); }
520 __DEVICE__ unsigned int __pm1(void) { return __nvvm_read_ptx_sreg_pm1(); }
521 __DEVICE__ unsigned int __pm2(void) { return __nvvm_read_ptx_sreg_pm2(); }
522 __DEVICE__ unsigned int __pm3(void) { return __nvvm_read_ptx_sreg_pm3(); }
523 __DEVICE__ int __popc(int __a) { return __nv_popc(__a); }
524 __DEVICE__ int __popcll(long long __a) { return __nv_popcll(__a); }
525 __DEVICE__ float __powf(float __a, float __b) {
526 return __nv_fast_powf(__a, __b);
529 // Parameter must have a known integer value.
530 #define __prof_trigger(__a) asm __volatile__("pmevent \t%0;" ::"i"(__a))
531 __DEVICE__ int __rhadd(int __a, int __b) { return __nv_rhadd(__a, __b); }
532 __DEVICE__ unsigned int __sad(int __a, int __b, unsigned int __c) {
533 return __nv_sad(__a, __b, __c);
535 __DEVICE__ float __saturatef(float __a) { return __nv_saturatef(__a); }
536 __DEVICE__ int __signbitd(double __a) { return __nv_signbitd(__a); }
537 __DEVICE__ int __signbitf(float __a) { return __nv_signbitf(__a); }
538 __DEVICE__ void __sincosf(float __a, float *__s, float *__c) {
539 return __nv_fast_sincosf(__a, __s, __c);
541 __DEVICE__ float __sinf(float __a) { return __nv_fast_sinf(__a); }
542 __DEVICE__ int __syncthreads_and(int __a) { return __nvvm_bar0_and(__a); }
543 __DEVICE__ int __syncthreads_count(int __a) { return __nvvm_bar0_popc(__a); }
544 __DEVICE__ int __syncthreads_or(int __a) { return __nvvm_bar0_or(__a); }
545 __DEVICE__ float __tanf(float __a) { return __nv_fast_tanf(__a); }
546 __DEVICE__ void __threadfence(void) { __nvvm_membar_gl(); }
547 __DEVICE__ void __threadfence_block(void) { __nvvm_membar_cta(); };
548 __DEVICE__ void __threadfence_system(void) { __nvvm_membar_sys(); };
549 __DEVICE__ void __trap(void) { asm volatile("trap;"); }
550 __DEVICE__ unsigned int __uAtomicAdd(unsigned int *__p, unsigned int __v) {
551 return __nvvm_atom_add_gen_i((int *)__p, __v);
553 __DEVICE__ unsigned int __uAtomicAdd_block(unsigned int *__p,
555 return __nvvm_atom_cta_add_gen_i((int *)__p, __v);
557 __DEVICE__ unsigned int __uAtomicAdd_system(unsigned int *__p,
559 return __nvvm_atom_sys_add_gen_i((int *)__p, __v);
561 __DEVICE__ unsigned int __uAtomicAnd(unsigned int *__p, unsigned int __v) {
562 return __nvvm_atom_and_gen_i((int *)__p, __v);
564 __DEVICE__ unsigned int __uAtomicAnd_block(unsigned int *__p,
566 return __nvvm_atom_cta_and_gen_i((int *)__p, __v);
568 __DEVICE__ unsigned int __uAtomicAnd_system(unsigned int *__p,
570 return __nvvm_atom_sys_and_gen_i((int *)__p, __v);
572 __DEVICE__ unsigned int __uAtomicCAS(unsigned int *__p, unsigned int __cmp,
574 return __nvvm_atom_cas_gen_i((int *)__p, __cmp, __v);
576 __DEVICE__ unsigned int
577 __uAtomicCAS_block(unsigned int *__p, unsigned int __cmp, unsigned int __v) {
578 return __nvvm_atom_cta_cas_gen_i((int *)__p, __cmp, __v);
580 __DEVICE__ unsigned int
581 __uAtomicCAS_system(unsigned int *__p, unsigned int __cmp, unsigned int __v) {
582 return __nvvm_atom_sys_cas_gen_i((int *)__p, __cmp, __v);
584 __DEVICE__ unsigned int __uAtomicDec(unsigned int *__p, unsigned int __v) {
585 return __nvvm_atom_dec_gen_ui(__p, __v);
587 __DEVICE__ unsigned int __uAtomicDec_block(unsigned int *__p,
589 return __nvvm_atom_cta_dec_gen_ui(__p, __v);
591 __DEVICE__ unsigned int __uAtomicDec_system(unsigned int *__p,
593 return __nvvm_atom_sys_dec_gen_ui(__p, __v);
595 __DEVICE__ unsigned int __uAtomicExch(unsigned int *__p, unsigned int __v) {
596 return __nvvm_atom_xchg_gen_i((int *)__p, __v);
598 __DEVICE__ unsigned int __uAtomicExch_block(unsigned int *__p,
600 return __nvvm_atom_cta_xchg_gen_i((int *)__p, __v);
602 __DEVICE__ unsigned int __uAtomicExch_system(unsigned int *__p,
604 return __nvvm_atom_sys_xchg_gen_i((int *)__p, __v);
606 __DEVICE__ unsigned int __uAtomicInc(unsigned int *__p, unsigned int __v) {
607 return __nvvm_atom_inc_gen_ui(__p, __v);
609 __DEVICE__ unsigned int __uAtomicInc_block(unsigned int *__p,
611 return __nvvm_atom_cta_inc_gen_ui(__p, __v);
613 __DEVICE__ unsigned int __uAtomicInc_system(unsigned int *__p,
615 return __nvvm_atom_sys_inc_gen_ui(__p, __v);
617 __DEVICE__ unsigned int __uAtomicMax(unsigned int *__p, unsigned int __v) {
618 return __nvvm_atom_max_gen_ui(__p, __v);
620 __DEVICE__ unsigned int __uAtomicMax_block(unsigned int *__p,
622 return __nvvm_atom_cta_max_gen_ui(__p, __v);
624 __DEVICE__ unsigned int __uAtomicMax_system(unsigned int *__p,
626 return __nvvm_atom_sys_max_gen_ui(__p, __v);
628 __DEVICE__ unsigned int __uAtomicMin(unsigned int *__p, unsigned int __v) {
629 return __nvvm_atom_min_gen_ui(__p, __v);
631 __DEVICE__ unsigned int __uAtomicMin_block(unsigned int *__p,
633 return __nvvm_atom_cta_min_gen_ui(__p, __v);
635 __DEVICE__ unsigned int __uAtomicMin_system(unsigned int *__p,
637 return __nvvm_atom_sys_min_gen_ui(__p, __v);
639 __DEVICE__ unsigned int __uAtomicOr(unsigned int *__p, unsigned int __v) {
640 return __nvvm_atom_or_gen_i((int *)__p, __v);
642 __DEVICE__ unsigned int __uAtomicOr_block(unsigned int *__p, unsigned int __v) {
643 return __nvvm_atom_cta_or_gen_i((int *)__p, __v);
645 __DEVICE__ unsigned int __uAtomicOr_system(unsigned int *__p,
647 return __nvvm_atom_sys_or_gen_i((int *)__p, __v);
649 __DEVICE__ unsigned int __uAtomicXor(unsigned int *__p, unsigned int __v) {
650 return __nvvm_atom_xor_gen_i((int *)__p, __v);
652 __DEVICE__ unsigned int __uAtomicXor_block(unsigned int *__p,
654 return __nvvm_atom_cta_xor_gen_i((int *)__p, __v);
656 __DEVICE__ unsigned int __uAtomicXor_system(unsigned int *__p,
658 return __nvvm_atom_sys_xor_gen_i((int *)__p, __v);
660 __DEVICE__ unsigned int __uhadd(unsigned int __a, unsigned int __b) {
661 return __nv_uhadd(__a, __b);
663 __DEVICE__ double __uint2double_rn(unsigned int __a) {
664 return __nv_uint2double_rn(__a);
666 __DEVICE__ float __uint2float_rd(unsigned int __a) {
667 return __nv_uint2float_rd(__a);
669 __DEVICE__ float __uint2float_rn(unsigned int __a) {
670 return __nv_uint2float_rn(__a);
672 __DEVICE__ float __uint2float_ru(unsigned int __a) {
673 return __nv_uint2float_ru(__a);
675 __DEVICE__ float __uint2float_rz(unsigned int __a) {
676 return __nv_uint2float_rz(__a);
678 __DEVICE__ float __uint_as_float(unsigned int __a) {
679 return __nv_uint_as_float(__a);
681 __DEVICE__ double __ull2double_rd(unsigned long long __a) {
682 return __nv_ull2double_rd(__a);
684 __DEVICE__ double __ull2double_rn(unsigned long long __a) {
685 return __nv_ull2double_rn(__a);
687 __DEVICE__ double __ull2double_ru(unsigned long long __a) {
688 return __nv_ull2double_ru(__a);
690 __DEVICE__ double __ull2double_rz(unsigned long long __a) {
691 return __nv_ull2double_rz(__a);
693 __DEVICE__ float __ull2float_rd(unsigned long long __a) {
694 return __nv_ull2float_rd(__a);
696 __DEVICE__ float __ull2float_rn(unsigned long long __a) {
697 return __nv_ull2float_rn(__a);
699 __DEVICE__ float __ull2float_ru(unsigned long long __a) {
700 return __nv_ull2float_ru(__a);
702 __DEVICE__ float __ull2float_rz(unsigned long long __a) {
703 return __nv_ull2float_rz(__a);
705 __DEVICE__ unsigned long long __ullAtomicAdd(unsigned long long *__p,
706 unsigned long long __v) {
707 return __nvvm_atom_add_gen_ll((long long *)__p, __v);
709 __DEVICE__ unsigned long long __ullAtomicAdd_block(unsigned long long *__p,
710 unsigned long long __v) {
711 return __nvvm_atom_cta_add_gen_ll((long long *)__p, __v);
713 __DEVICE__ unsigned long long __ullAtomicAdd_system(unsigned long long *__p,
714 unsigned long long __v) {
715 return __nvvm_atom_sys_add_gen_ll((long long *)__p, __v);
717 __DEVICE__ unsigned long long __ullAtomicAnd(unsigned long long *__p,
718 unsigned long long __v) {
719 return __nvvm_atom_and_gen_ll((long long *)__p, __v);
721 __DEVICE__ unsigned long long __ullAtomicAnd_block(unsigned long long *__p,
722 unsigned long long __v) {
723 return __nvvm_atom_cta_and_gen_ll((long long *)__p, __v);
725 __DEVICE__ unsigned long long __ullAtomicAnd_system(unsigned long long *__p,
726 unsigned long long __v) {
727 return __nvvm_atom_sys_and_gen_ll((long long *)__p, __v);
729 __DEVICE__ unsigned long long __ullAtomicCAS(unsigned long long *__p,
730 unsigned long long __cmp,
731 unsigned long long __v) {
732 return __nvvm_atom_cas_gen_ll((long long *)__p, __cmp, __v);
734 __DEVICE__ unsigned long long __ullAtomicCAS_block(unsigned long long *__p,
735 unsigned long long __cmp,
736 unsigned long long __v) {
737 return __nvvm_atom_cta_cas_gen_ll((long long *)__p, __cmp, __v);
739 __DEVICE__ unsigned long long __ullAtomicCAS_system(unsigned long long *__p,
740 unsigned long long __cmp,
741 unsigned long long __v) {
742 return __nvvm_atom_sys_cas_gen_ll((long long *)__p, __cmp, __v);
744 __DEVICE__ unsigned long long __ullAtomicExch(unsigned long long *__p,
745 unsigned long long __v) {
746 return __nvvm_atom_xchg_gen_ll((long long *)__p, __v);
748 __DEVICE__ unsigned long long __ullAtomicExch_block(unsigned long long *__p,
749 unsigned long long __v) {
750 return __nvvm_atom_cta_xchg_gen_ll((long long *)__p, __v);
752 __DEVICE__ unsigned long long __ullAtomicExch_system(unsigned long long *__p,
753 unsigned long long __v) {
754 return __nvvm_atom_sys_xchg_gen_ll((long long *)__p, __v);
756 __DEVICE__ unsigned long long __ullAtomicMax(unsigned long long *__p,
757 unsigned long long __v) {
758 return __nvvm_atom_max_gen_ull(__p, __v);
760 __DEVICE__ unsigned long long __ullAtomicMax_block(unsigned long long *__p,
761 unsigned long long __v) {
762 return __nvvm_atom_cta_max_gen_ull(__p, __v);
764 __DEVICE__ unsigned long long __ullAtomicMax_system(unsigned long long *__p,
765 unsigned long long __v) {
766 return __nvvm_atom_sys_max_gen_ull(__p, __v);
768 __DEVICE__ unsigned long long __ullAtomicMin(unsigned long long *__p,
769 unsigned long long __v) {
770 return __nvvm_atom_min_gen_ull(__p, __v);
772 __DEVICE__ unsigned long long __ullAtomicMin_block(unsigned long long *__p,
773 unsigned long long __v) {
774 return __nvvm_atom_cta_min_gen_ull(__p, __v);
776 __DEVICE__ unsigned long long __ullAtomicMin_system(unsigned long long *__p,
777 unsigned long long __v) {
778 return __nvvm_atom_sys_min_gen_ull(__p, __v);
780 __DEVICE__ unsigned long long __ullAtomicOr(unsigned long long *__p,
781 unsigned long long __v) {
782 return __nvvm_atom_or_gen_ll((long long *)__p, __v);
784 __DEVICE__ unsigned long long __ullAtomicOr_block(unsigned long long *__p,
785 unsigned long long __v) {
786 return __nvvm_atom_cta_or_gen_ll((long long *)__p, __v);
788 __DEVICE__ unsigned long long __ullAtomicOr_system(unsigned long long *__p,
789 unsigned long long __v) {
790 return __nvvm_atom_sys_or_gen_ll((long long *)__p, __v);
792 __DEVICE__ unsigned long long __ullAtomicXor(unsigned long long *__p,
793 unsigned long long __v) {
794 return __nvvm_atom_xor_gen_ll((long long *)__p, __v);
796 __DEVICE__ unsigned long long __ullAtomicXor_block(unsigned long long *__p,
797 unsigned long long __v) {
798 return __nvvm_atom_cta_xor_gen_ll((long long *)__p, __v);
800 __DEVICE__ unsigned long long __ullAtomicXor_system(unsigned long long *__p,
801 unsigned long long __v) {
802 return __nvvm_atom_sys_xor_gen_ll((long long *)__p, __v);
804 __DEVICE__ unsigned int __umul24(unsigned int __a, unsigned int __b) {
805 return __nv_umul24(__a, __b);
807 __DEVICE__ unsigned long long __umul64hi(unsigned long long __a,
808 unsigned long long __b) {
809 return __nv_umul64hi(__a, __b);
811 __DEVICE__ unsigned int __umulhi(unsigned int __a, unsigned int __b) {
812 return __nv_umulhi(__a, __b);
814 __DEVICE__ unsigned int __urhadd(unsigned int __a, unsigned int __b) {
815 return __nv_urhadd(__a, __b);
817 __DEVICE__ unsigned int __usad(unsigned int __a, unsigned int __b,
819 return __nv_usad(__a, __b, __c);
822 #if CUDA_VERSION >= 9000 && CUDA_VERSION < 9020
823 __DEVICE__ unsigned int __vabs2(unsigned int __a) { return __nv_vabs2(__a); }
824 __DEVICE__ unsigned int __vabs4(unsigned int __a) { return __nv_vabs4(__a); }
825 __DEVICE__ unsigned int __vabsdiffs2(unsigned int __a, unsigned int __b) {
826 return __nv_vabsdiffs2(__a, __b);
828 __DEVICE__ unsigned int __vabsdiffs4(unsigned int __a, unsigned int __b) {
829 return __nv_vabsdiffs4(__a, __b);
831 __DEVICE__ unsigned int __vabsdiffu2(unsigned int __a, unsigned int __b) {
832 return __nv_vabsdiffu2(__a, __b);
834 __DEVICE__ unsigned int __vabsdiffu4(unsigned int __a, unsigned int __b) {
835 return __nv_vabsdiffu4(__a, __b);
837 __DEVICE__ unsigned int __vabsss2(unsigned int __a) {
838 return __nv_vabsss2(__a);
840 __DEVICE__ unsigned int __vabsss4(unsigned int __a) {
841 return __nv_vabsss4(__a);
843 __DEVICE__ unsigned int __vadd2(unsigned int __a, unsigned int __b) {
844 return __nv_vadd2(__a, __b);
846 __DEVICE__ unsigned int __vadd4(unsigned int __a, unsigned int __b) {
847 return __nv_vadd4(__a, __b);
849 __DEVICE__ unsigned int __vaddss2(unsigned int __a, unsigned int __b) {
850 return __nv_vaddss2(__a, __b);
852 __DEVICE__ unsigned int __vaddss4(unsigned int __a, unsigned int __b) {
853 return __nv_vaddss4(__a, __b);
855 __DEVICE__ unsigned int __vaddus2(unsigned int __a, unsigned int __b) {
856 return __nv_vaddus2(__a, __b);
858 __DEVICE__ unsigned int __vaddus4(unsigned int __a, unsigned int __b) {
859 return __nv_vaddus4(__a, __b);
861 __DEVICE__ unsigned int __vavgs2(unsigned int __a, unsigned int __b) {
862 return __nv_vavgs2(__a, __b);
864 __DEVICE__ unsigned int __vavgs4(unsigned int __a, unsigned int __b) {
865 return __nv_vavgs4(__a, __b);
867 __DEVICE__ unsigned int __vavgu2(unsigned int __a, unsigned int __b) {
868 return __nv_vavgu2(__a, __b);
870 __DEVICE__ unsigned int __vavgu4(unsigned int __a, unsigned int __b) {
871 return __nv_vavgu4(__a, __b);
873 __DEVICE__ unsigned int __vcmpeq2(unsigned int __a, unsigned int __b) {
874 return __nv_vcmpeq2(__a, __b);
876 __DEVICE__ unsigned int __vcmpeq4(unsigned int __a, unsigned int __b) {
877 return __nv_vcmpeq4(__a, __b);
879 __DEVICE__ unsigned int __vcmpges2(unsigned int __a, unsigned int __b) {
880 return __nv_vcmpges2(__a, __b);
882 __DEVICE__ unsigned int __vcmpges4(unsigned int __a, unsigned int __b) {
883 return __nv_vcmpges4(__a, __b);
885 __DEVICE__ unsigned int __vcmpgeu2(unsigned int __a, unsigned int __b) {
886 return __nv_vcmpgeu2(__a, __b);
888 __DEVICE__ unsigned int __vcmpgeu4(unsigned int __a, unsigned int __b) {
889 return __nv_vcmpgeu4(__a, __b);
891 __DEVICE__ unsigned int __vcmpgts2(unsigned int __a, unsigned int __b) {
892 return __nv_vcmpgts2(__a, __b);
894 __DEVICE__ unsigned int __vcmpgts4(unsigned int __a, unsigned int __b) {
895 return __nv_vcmpgts4(__a, __b);
897 __DEVICE__ unsigned int __vcmpgtu2(unsigned int __a, unsigned int __b) {
898 return __nv_vcmpgtu2(__a, __b);
900 __DEVICE__ unsigned int __vcmpgtu4(unsigned int __a, unsigned int __b) {
901 return __nv_vcmpgtu4(__a, __b);
903 __DEVICE__ unsigned int __vcmples2(unsigned int __a, unsigned int __b) {
904 return __nv_vcmples2(__a, __b);
906 __DEVICE__ unsigned int __vcmples4(unsigned int __a, unsigned int __b) {
907 return __nv_vcmples4(__a, __b);
909 __DEVICE__ unsigned int __vcmpleu2(unsigned int __a, unsigned int __b) {
910 return __nv_vcmpleu2(__a, __b);
912 __DEVICE__ unsigned int __vcmpleu4(unsigned int __a, unsigned int __b) {
913 return __nv_vcmpleu4(__a, __b);
915 __DEVICE__ unsigned int __vcmplts2(unsigned int __a, unsigned int __b) {
916 return __nv_vcmplts2(__a, __b);
918 __DEVICE__ unsigned int __vcmplts4(unsigned int __a, unsigned int __b) {
919 return __nv_vcmplts4(__a, __b);
921 __DEVICE__ unsigned int __vcmpltu2(unsigned int __a, unsigned int __b) {
922 return __nv_vcmpltu2(__a, __b);
924 __DEVICE__ unsigned int __vcmpltu4(unsigned int __a, unsigned int __b) {
925 return __nv_vcmpltu4(__a, __b);
927 __DEVICE__ unsigned int __vcmpne2(unsigned int __a, unsigned int __b) {
928 return __nv_vcmpne2(__a, __b);
930 __DEVICE__ unsigned int __vcmpne4(unsigned int __a, unsigned int __b) {
931 return __nv_vcmpne4(__a, __b);
933 __DEVICE__ unsigned int __vhaddu2(unsigned int __a, unsigned int __b) {
934 return __nv_vhaddu2(__a, __b);
936 __DEVICE__ unsigned int __vhaddu4(unsigned int __a, unsigned int __b) {
937 return __nv_vhaddu4(__a, __b);
939 __DEVICE__ unsigned int __vmaxs2(unsigned int __a, unsigned int __b) {
940 return __nv_vmaxs2(__a, __b);
942 __DEVICE__ unsigned int __vmaxs4(unsigned int __a, unsigned int __b) {
943 return __nv_vmaxs4(__a, __b);
945 __DEVICE__ unsigned int __vmaxu2(unsigned int __a, unsigned int __b) {
946 return __nv_vmaxu2(__a, __b);
948 __DEVICE__ unsigned int __vmaxu4(unsigned int __a, unsigned int __b) {
949 return __nv_vmaxu4(__a, __b);
951 __DEVICE__ unsigned int __vmins2(unsigned int __a, unsigned int __b) {
952 return __nv_vmins2(__a, __b);
954 __DEVICE__ unsigned int __vmins4(unsigned int __a, unsigned int __b) {
955 return __nv_vmins4(__a, __b);
957 __DEVICE__ unsigned int __vminu2(unsigned int __a, unsigned int __b) {
958 return __nv_vminu2(__a, __b);
960 __DEVICE__ unsigned int __vminu4(unsigned int __a, unsigned int __b) {
961 return __nv_vminu4(__a, __b);
963 __DEVICE__ unsigned int __vneg2(unsigned int __a) { return __nv_vneg2(__a); }
964 __DEVICE__ unsigned int __vneg4(unsigned int __a) { return __nv_vneg4(__a); }
965 __DEVICE__ unsigned int __vnegss2(unsigned int __a) {
966 return __nv_vnegss2(__a);
968 __DEVICE__ unsigned int __vnegss4(unsigned int __a) {
969 return __nv_vnegss4(__a);
971 __DEVICE__ unsigned int __vsads2(unsigned int __a, unsigned int __b) {
972 return __nv_vsads2(__a, __b);
974 __DEVICE__ unsigned int __vsads4(unsigned int __a, unsigned int __b) {
975 return __nv_vsads4(__a, __b);
977 __DEVICE__ unsigned int __vsadu2(unsigned int __a, unsigned int __b) {
978 return __nv_vsadu2(__a, __b);
980 __DEVICE__ unsigned int __vsadu4(unsigned int __a, unsigned int __b) {
981 return __nv_vsadu4(__a, __b);
983 __DEVICE__ unsigned int __vseteq2(unsigned int __a, unsigned int __b) {
984 return __nv_vseteq2(__a, __b);
986 __DEVICE__ unsigned int __vseteq4(unsigned int __a, unsigned int __b) {
987 return __nv_vseteq4(__a, __b);
989 __DEVICE__ unsigned int __vsetges2(unsigned int __a, unsigned int __b) {
990 return __nv_vsetges2(__a, __b);
992 __DEVICE__ unsigned int __vsetges4(unsigned int __a, unsigned int __b) {
993 return __nv_vsetges4(__a, __b);
995 __DEVICE__ unsigned int __vsetgeu2(unsigned int __a, unsigned int __b) {
996 return __nv_vsetgeu2(__a, __b);
998 __DEVICE__ unsigned int __vsetgeu4(unsigned int __a, unsigned int __b) {
999 return __nv_vsetgeu4(__a, __b);
1001 __DEVICE__ unsigned int __vsetgts2(unsigned int __a, unsigned int __b) {
1002 return __nv_vsetgts2(__a, __b);
1004 __DEVICE__ unsigned int __vsetgts4(unsigned int __a, unsigned int __b) {
1005 return __nv_vsetgts4(__a, __b);
1007 __DEVICE__ unsigned int __vsetgtu2(unsigned int __a, unsigned int __b) {
1008 return __nv_vsetgtu2(__a, __b);
1010 __DEVICE__ unsigned int __vsetgtu4(unsigned int __a, unsigned int __b) {
1011 return __nv_vsetgtu4(__a, __b);
1013 __DEVICE__ unsigned int __vsetles2(unsigned int __a, unsigned int __b) {
1014 return __nv_vsetles2(__a, __b);
1016 __DEVICE__ unsigned int __vsetles4(unsigned int __a, unsigned int __b) {
1017 return __nv_vsetles4(__a, __b);
1019 __DEVICE__ unsigned int __vsetleu2(unsigned int __a, unsigned int __b) {
1020 return __nv_vsetleu2(__a, __b);
1022 __DEVICE__ unsigned int __vsetleu4(unsigned int __a, unsigned int __b) {
1023 return __nv_vsetleu4(__a, __b);
1025 __DEVICE__ unsigned int __vsetlts2(unsigned int __a, unsigned int __b) {
1026 return __nv_vsetlts2(__a, __b);
1028 __DEVICE__ unsigned int __vsetlts4(unsigned int __a, unsigned int __b) {
1029 return __nv_vsetlts4(__a, __b);
1031 __DEVICE__ unsigned int __vsetltu2(unsigned int __a, unsigned int __b) {
1032 return __nv_vsetltu2(__a, __b);
1034 __DEVICE__ unsigned int __vsetltu4(unsigned int __a, unsigned int __b) {
1035 return __nv_vsetltu4(__a, __b);
1037 __DEVICE__ unsigned int __vsetne2(unsigned int __a, unsigned int __b) {
1038 return __nv_vsetne2(__a, __b);
1040 __DEVICE__ unsigned int __vsetne4(unsigned int __a, unsigned int __b) {
1041 return __nv_vsetne4(__a, __b);
1043 __DEVICE__ unsigned int __vsub2(unsigned int __a, unsigned int __b) {
1044 return __nv_vsub2(__a, __b);
1046 __DEVICE__ unsigned int __vsub4(unsigned int __a, unsigned int __b) {
1047 return __nv_vsub4(__a, __b);
1049 __DEVICE__ unsigned int __vsubss2(unsigned int __a, unsigned int __b) {
1050 return __nv_vsubss2(__a, __b);
1052 __DEVICE__ unsigned int __vsubss4(unsigned int __a, unsigned int __b) {
1053 return __nv_vsubss4(__a, __b);
1055 __DEVICE__ unsigned int __vsubus2(unsigned int __a, unsigned int __b) {
1056 return __nv_vsubus2(__a, __b);
1058 __DEVICE__ unsigned int __vsubus4(unsigned int __a, unsigned int __b) {
1059 return __nv_vsubus4(__a, __b);
1061 #else // CUDA_VERSION >= 9020
1062 // CUDA no longer provides inline assembly (or bitcode) implementation of these
1063 // functions, so we have to reimplment them. The implementation is naive and is
1064 // not optimized for performance.
1066 // Helper function to convert N-bit boolean subfields into all-0 or all-1.
1067 // E.g. __bool2mask(0x01000100,8) -> 0xff00ff00
1068 // __bool2mask(0x00010000,16) -> 0xffff0000
1069 __DEVICE__ unsigned int __bool2mask(unsigned int __a, int shift) {
1070 return (__a << shift) - __a;
1072 __DEVICE__ unsigned int __vabs2(unsigned int __a) {
1074 asm("vabsdiff2.s32.s32.s32 %0,%1,%2,%3;"
1076 : "r"(__a), "r"(0), "r"(0));
1079 __DEVICE__ unsigned int __vabs4(unsigned int __a) {
1081 asm("vabsdiff4.s32.s32.s32 %0,%1,%2,%3;"
1083 : "r"(__a), "r"(0), "r"(0));
1086 __DEVICE__ unsigned int __vabsdiffs2(unsigned int __a, unsigned int __b) {
1088 asm("vabsdiff2.s32.s32.s32 %0,%1,%2,%3;"
1090 : "r"(__a), "r"(__b), "r"(0));
1094 __DEVICE__ unsigned int __vabsdiffs4(unsigned int __a, unsigned int __b) {
1096 asm("vabsdiff4.s32.s32.s32 %0,%1,%2,%3;"
1098 : "r"(__a), "r"(__b), "r"(0));
1101 __DEVICE__ unsigned int __vabsdiffu2(unsigned int __a, unsigned int __b) {
1103 asm("vabsdiff2.u32.u32.u32 %0,%1,%2,%3;"
1105 : "r"(__a), "r"(__b), "r"(0));
1108 __DEVICE__ unsigned int __vabsdiffu4(unsigned int __a, unsigned int __b) {
1110 asm("vabsdiff4.u32.u32.u32 %0,%1,%2,%3;"
1112 : "r"(__a), "r"(__b), "r"(0));
1115 __DEVICE__ unsigned int __vabsss2(unsigned int __a) {
1117 asm("vabsdiff2.s32.s32.s32.sat %0,%1,%2,%3;"
1119 : "r"(__a), "r"(0), "r"(0));
1122 __DEVICE__ unsigned int __vabsss4(unsigned int __a) {
1124 asm("vabsdiff4.s32.s32.s32.sat %0,%1,%2,%3;"
1126 : "r"(__a), "r"(0), "r"(0));
1129 __DEVICE__ unsigned int __vadd2(unsigned int __a, unsigned int __b) {
1131 asm("vadd2.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1134 __DEVICE__ unsigned int __vadd4(unsigned int __a, unsigned int __b) {
1136 asm("vadd4.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1139 __DEVICE__ unsigned int __vaddss2(unsigned int __a, unsigned int __b) {
1141 asm("vadd2.s32.s32.s32.sat %0,%1,%2,%3;"
1143 : "r"(__a), "r"(__b), "r"(0));
1146 __DEVICE__ unsigned int __vaddss4(unsigned int __a, unsigned int __b) {
1148 asm("vadd4.s32.s32.s32.sat %0,%1,%2,%3;"
1150 : "r"(__a), "r"(__b), "r"(0));
1153 __DEVICE__ unsigned int __vaddus2(unsigned int __a, unsigned int __b) {
1155 asm("vadd2.u32.u32.u32.sat %0,%1,%2,%3;"
1157 : "r"(__a), "r"(__b), "r"(0));
1160 __DEVICE__ unsigned int __vaddus4(unsigned int __a, unsigned int __b) {
1162 asm("vadd4.u32.u32.u32.sat %0,%1,%2,%3;"
1164 : "r"(__a), "r"(__b), "r"(0));
1167 __DEVICE__ unsigned int __vavgs2(unsigned int __a, unsigned int __b) {
1169 asm("vavrg2.s32.s32.s32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1172 __DEVICE__ unsigned int __vavgs4(unsigned int __a, unsigned int __b) {
1174 asm("vavrg4.s32.s32.s32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1177 __DEVICE__ unsigned int __vavgu2(unsigned int __a, unsigned int __b) {
1179 asm("vavrg2.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1182 __DEVICE__ unsigned int __vavgu4(unsigned int __a, unsigned int __b) {
1184 asm("vavrg4.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1187 __DEVICE__ unsigned int __vseteq2(unsigned int __a, unsigned int __b) {
1189 asm("vset2.u32.u32.eq %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1192 __DEVICE__ unsigned int __vcmpeq2(unsigned int __a, unsigned int __b) {
1193 return __bool2mask(__vseteq2(__a, __b), 16);
1195 __DEVICE__ unsigned int __vseteq4(unsigned int __a, unsigned int __b) {
1197 asm("vset4.u32.u32.eq %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1200 __DEVICE__ unsigned int __vcmpeq4(unsigned int __a, unsigned int __b) {
1201 return __bool2mask(__vseteq4(__a, __b), 8);
1203 __DEVICE__ unsigned int __vsetges2(unsigned int __a, unsigned int __b) {
1205 asm("vset2.s32.s32.ge %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1208 __DEVICE__ unsigned int __vcmpges2(unsigned int __a, unsigned int __b) {
1209 return __bool2mask(__vsetges2(__a, __b), 16);
1211 __DEVICE__ unsigned int __vsetges4(unsigned int __a, unsigned int __b) {
1213 asm("vset4.s32.s32.ge %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1216 __DEVICE__ unsigned int __vcmpges4(unsigned int __a, unsigned int __b) {
1217 return __bool2mask(__vsetges4(__a, __b), 8);
1219 __DEVICE__ unsigned int __vsetgeu2(unsigned int __a, unsigned int __b) {
1221 asm("vset2.u32.u32.ge %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1224 __DEVICE__ unsigned int __vcmpgeu2(unsigned int __a, unsigned int __b) {
1225 return __bool2mask(__vsetgeu2(__a, __b), 16);
1227 __DEVICE__ unsigned int __vsetgeu4(unsigned int __a, unsigned int __b) {
1229 asm("vset4.u32.u32.ge %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1232 __DEVICE__ unsigned int __vcmpgeu4(unsigned int __a, unsigned int __b) {
1233 return __bool2mask(__vsetgeu4(__a, __b), 8);
1235 __DEVICE__ unsigned int __vsetgts2(unsigned int __a, unsigned int __b) {
1237 asm("vset2.s32.s32.gt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1240 __DEVICE__ unsigned int __vcmpgts2(unsigned int __a, unsigned int __b) {
1241 return __bool2mask(__vsetgts2(__a, __b), 16);
1243 __DEVICE__ unsigned int __vsetgts4(unsigned int __a, unsigned int __b) {
1245 asm("vset4.s32.s32.gt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1248 __DEVICE__ unsigned int __vcmpgts4(unsigned int __a, unsigned int __b) {
1249 return __bool2mask(__vsetgts4(__a, __b), 8);
1251 __DEVICE__ unsigned int __vsetgtu2(unsigned int __a, unsigned int __b) {
1253 asm("vset2.u32.u32.gt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1256 __DEVICE__ unsigned int __vcmpgtu2(unsigned int __a, unsigned int __b) {
1257 return __bool2mask(__vsetgtu2(__a, __b), 16);
1259 __DEVICE__ unsigned int __vsetgtu4(unsigned int __a, unsigned int __b) {
1261 asm("vset4.u32.u32.gt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1264 __DEVICE__ unsigned int __vcmpgtu4(unsigned int __a, unsigned int __b) {
1265 return __bool2mask(__vsetgtu4(__a, __b), 8);
1267 __DEVICE__ unsigned int __vsetles2(unsigned int __a, unsigned int __b) {
1269 asm("vset2.s32.s32.le %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1272 __DEVICE__ unsigned int __vcmples2(unsigned int __a, unsigned int __b) {
1273 return __bool2mask(__vsetles2(__a, __b), 16);
1275 __DEVICE__ unsigned int __vsetles4(unsigned int __a, unsigned int __b) {
1277 asm("vset4.s32.s32.le %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1280 __DEVICE__ unsigned int __vcmples4(unsigned int __a, unsigned int __b) {
1281 return __bool2mask(__vsetles4(__a, __b), 8);
1283 __DEVICE__ unsigned int __vsetleu2(unsigned int __a, unsigned int __b) {
1285 asm("vset2.u32.u32.le %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1288 __DEVICE__ unsigned int __vcmpleu2(unsigned int __a, unsigned int __b) {
1289 return __bool2mask(__vsetleu2(__a, __b), 16);
1291 __DEVICE__ unsigned int __vsetleu4(unsigned int __a, unsigned int __b) {
1293 asm("vset4.u32.u32.le %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1296 __DEVICE__ unsigned int __vcmpleu4(unsigned int __a, unsigned int __b) {
1297 return __bool2mask(__vsetleu4(__a, __b), 8);
1299 __DEVICE__ unsigned int __vsetlts2(unsigned int __a, unsigned int __b) {
1301 asm("vset2.s32.s32.lt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1304 __DEVICE__ unsigned int __vcmplts2(unsigned int __a, unsigned int __b) {
1305 return __bool2mask(__vsetlts2(__a, __b), 16);
1307 __DEVICE__ unsigned int __vsetlts4(unsigned int __a, unsigned int __b) {
1309 asm("vset4.s32.s32.lt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1312 __DEVICE__ unsigned int __vcmplts4(unsigned int __a, unsigned int __b) {
1313 return __bool2mask(__vsetlts4(__a, __b), 8);
1315 __DEVICE__ unsigned int __vsetltu2(unsigned int __a, unsigned int __b) {
1317 asm("vset2.u32.u32.lt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1320 __DEVICE__ unsigned int __vcmpltu2(unsigned int __a, unsigned int __b) {
1321 return __bool2mask(__vsetltu2(__a, __b), 16);
1323 __DEVICE__ unsigned int __vsetltu4(unsigned int __a, unsigned int __b) {
1325 asm("vset4.u32.u32.lt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1328 __DEVICE__ unsigned int __vcmpltu4(unsigned int __a, unsigned int __b) {
1329 return __bool2mask(__vsetltu4(__a, __b), 8);
1331 __DEVICE__ unsigned int __vsetne2(unsigned int __a, unsigned int __b) {
1333 asm("vset2.u32.u32.ne %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1336 __DEVICE__ unsigned int __vcmpne2(unsigned int __a, unsigned int __b) {
1337 return __bool2mask(__vsetne2(__a, __b), 16);
1339 __DEVICE__ unsigned int __vsetne4(unsigned int __a, unsigned int __b) {
1341 asm("vset4.u32.u32.ne %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1344 __DEVICE__ unsigned int __vcmpne4(unsigned int __a, unsigned int __b) {
1345 return __bool2mask(__vsetne4(__a, __b), 8);
1348 // Based on ITEM 23 in AIM-239: http://dspace.mit.edu/handle/1721.1/6086
1349 // (a & b) + (a | b) = a + b = (a ^ b) + 2 * (a & b) =>
1350 // (a + b) / 2 = ((a ^ b) >> 1) + (a & b)
1351 // To operate on multiple sub-elements we need to make sure to mask out bits
1352 // that crossed over into adjacent elements during the shift.
1353 __DEVICE__ unsigned int __vhaddu2(unsigned int __a, unsigned int __b) {
1354 return (((__a ^ __b) >> 1) & ~0x80008000u) + (__a & __b);
1356 __DEVICE__ unsigned int __vhaddu4(unsigned int __a, unsigned int __b) {
1357 return (((__a ^ __b) >> 1) & ~0x80808080u) + (__a & __b);
1360 __DEVICE__ unsigned int __vmaxs2(unsigned int __a, unsigned int __b) {
1362 if ((__a & 0x8000) && (__b & 0x8000)) {
1363 // Work around a bug in ptxas which produces invalid result if low element
1365 unsigned mask = __vcmpgts2(__a, __b);
1366 r = (__a & mask) | (__b & ~mask);
1368 asm("vmax2.s32.s32.s32 %0,%1,%2,%3;"
1370 : "r"(__a), "r"(__b), "r"(0));
1374 __DEVICE__ unsigned int __vmaxs4(unsigned int __a, unsigned int __b) {
1376 asm("vmax4.s32.s32.s32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1379 __DEVICE__ unsigned int __vmaxu2(unsigned int __a, unsigned int __b) {
1381 asm("vmax2.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1384 __DEVICE__ unsigned int __vmaxu4(unsigned int __a, unsigned int __b) {
1386 asm("vmax4.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1389 __DEVICE__ unsigned int __vmins2(unsigned int __a, unsigned int __b) {
1391 asm("vmin2.s32.s32.s32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1394 __DEVICE__ unsigned int __vmins4(unsigned int __a, unsigned int __b) {
1396 asm("vmin4.s32.s32.s32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1399 __DEVICE__ unsigned int __vminu2(unsigned int __a, unsigned int __b) {
1401 asm("vmin2.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1404 __DEVICE__ unsigned int __vminu4(unsigned int __a, unsigned int __b) {
1406 asm("vmin4.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1409 __DEVICE__ unsigned int __vsads2(unsigned int __a, unsigned int __b) {
1411 asm("vabsdiff2.s32.s32.s32.add %0,%1,%2,%3;"
1413 : "r"(__a), "r"(__b), "r"(0));
1416 __DEVICE__ unsigned int __vsads4(unsigned int __a, unsigned int __b) {
1418 asm("vabsdiff4.s32.s32.s32.add %0,%1,%2,%3;"
1420 : "r"(__a), "r"(__b), "r"(0));
1423 __DEVICE__ unsigned int __vsadu2(unsigned int __a, unsigned int __b) {
1425 asm("vabsdiff2.u32.u32.u32.add %0,%1,%2,%3;"
1427 : "r"(__a), "r"(__b), "r"(0));
1430 __DEVICE__ unsigned int __vsadu4(unsigned int __a, unsigned int __b) {
1432 asm("vabsdiff4.u32.u32.u32.add %0,%1,%2,%3;"
1434 : "r"(__a), "r"(__b), "r"(0));
1438 __DEVICE__ unsigned int __vsub2(unsigned int __a, unsigned int __b) {
1440 asm("vsub2.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1443 __DEVICE__ unsigned int __vneg2(unsigned int __a) { return __vsub2(0, __a); }
1445 __DEVICE__ unsigned int __vsub4(unsigned int __a, unsigned int __b) {
1447 asm("vsub4.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0));
1450 __DEVICE__ unsigned int __vneg4(unsigned int __a) { return __vsub4(0, __a); }
1451 __DEVICE__ unsigned int __vsubss2(unsigned int __a, unsigned int __b) {
1453 asm("vsub2.s32.s32.s32.sat %0,%1,%2,%3;"
1455 : "r"(__a), "r"(__b), "r"(0));
1458 __DEVICE__ unsigned int __vnegss2(unsigned int __a) {
1459 return __vsubss2(0, __a);
1461 __DEVICE__ unsigned int __vsubss4(unsigned int __a, unsigned int __b) {
1463 asm("vsub4.s32.s32.s32.sat %0,%1,%2,%3;"
1465 : "r"(__a), "r"(__b), "r"(0));
1468 __DEVICE__ unsigned int __vnegss4(unsigned int __a) {
1469 return __vsubss4(0, __a);
1471 __DEVICE__ unsigned int __vsubus2(unsigned int __a, unsigned int __b) {
1473 asm("vsub2.u32.u32.u32.sat %0,%1,%2,%3;"
1475 : "r"(__a), "r"(__b), "r"(0));
1478 __DEVICE__ unsigned int __vsubus4(unsigned int __a, unsigned int __b) {
1480 asm("vsub4.u32.u32.u32.sat %0,%1,%2,%3;"
1482 : "r"(__a), "r"(__b), "r"(0));
1485 #endif // CUDA_VERSION >= 9020
1486 __DEVICE__ int abs(int __a) __NOEXCEPT { return __nv_abs(__a); }
1487 __DEVICE__ double fabs(double __a) __NOEXCEPT { return __nv_fabs(__a); }
1488 __DEVICE__ double acos(double __a) { return __nv_acos(__a); }
1489 __DEVICE__ float acosf(float __a) { return __nv_acosf(__a); }
1490 __DEVICE__ double acosh(double __a) { return __nv_acosh(__a); }
1491 __DEVICE__ float acoshf(float __a) { return __nv_acoshf(__a); }
1492 __DEVICE__ double asin(double __a) { return __nv_asin(__a); }
1493 __DEVICE__ float asinf(float __a) { return __nv_asinf(__a); }
1494 __DEVICE__ double asinh(double __a) { return __nv_asinh(__a); }
1495 __DEVICE__ float asinhf(float __a) { return __nv_asinhf(__a); }
1496 __DEVICE__ double atan(double __a) { return __nv_atan(__a); }
1497 __DEVICE__ double atan2(double __a, double __b) { return __nv_atan2(__a, __b); }
1498 __DEVICE__ float atan2f(float __a, float __b) { return __nv_atan2f(__a, __b); }
1499 __DEVICE__ float atanf(float __a) { return __nv_atanf(__a); }
1500 __DEVICE__ double atanh(double __a) { return __nv_atanh(__a); }
1501 __DEVICE__ float atanhf(float __a) { return __nv_atanhf(__a); }
1502 __DEVICE__ double cbrt(double __a) { return __nv_cbrt(__a); }
1503 __DEVICE__ float cbrtf(float __a) { return __nv_cbrtf(__a); }
1504 __DEVICE__ double ceil(double __a) { return __nv_ceil(__a); }
1505 __DEVICE__ float ceilf(float __a) { return __nv_ceilf(__a); }
1507 __DEVICE__ int clock() { return __nvvm_read_ptx_sreg_clock(); }
1508 __DEVICE__ long long clock64() { return __nvvm_read_ptx_sreg_clock64(); }
1510 __DEVICE__ double copysign(double __a, double __b) {
1511 return __nv_copysign(__a, __b);
1513 __DEVICE__ float copysignf(float __a, float __b) {
1514 return __nv_copysignf(__a, __b);
1516 __DEVICE__ double cos(double __a) { return __nv_cos(__a); }
1517 __DEVICE__ float cosf(float __a) {
1518 return __FAST_OR_SLOW(__nv_fast_cosf, __nv_cosf)(__a);
1520 __DEVICE__ double cosh(double __a) { return __nv_cosh(__a); }
1521 __DEVICE__ float coshf(float __a) { return __nv_coshf(__a); }
1522 __DEVICE__ double cospi(double __a) { return __nv_cospi(__a); }
1523 __DEVICE__ float cospif(float __a) { return __nv_cospif(__a); }
1524 __DEVICE__ double cyl_bessel_i0(double __a) { return __nv_cyl_bessel_i0(__a); }
1525 __DEVICE__ float cyl_bessel_i0f(float __a) { return __nv_cyl_bessel_i0f(__a); }
1526 __DEVICE__ double cyl_bessel_i1(double __a) { return __nv_cyl_bessel_i1(__a); }
1527 __DEVICE__ float cyl_bessel_i1f(float __a) { return __nv_cyl_bessel_i1f(__a); }
1528 __DEVICE__ double erf(double __a) { return __nv_erf(__a); }
1529 __DEVICE__ double erfc(double __a) { return __nv_erfc(__a); }
1530 __DEVICE__ float erfcf(float __a) { return __nv_erfcf(__a); }
1531 __DEVICE__ double erfcinv(double __a) { return __nv_erfcinv(__a); }
1532 __DEVICE__ float erfcinvf(float __a) { return __nv_erfcinvf(__a); }
1533 __DEVICE__ double erfcx(double __a) { return __nv_erfcx(__a); }
1534 __DEVICE__ float erfcxf(float __a) { return __nv_erfcxf(__a); }
1535 __DEVICE__ float erff(float __a) { return __nv_erff(__a); }
1536 __DEVICE__ double erfinv(double __a) { return __nv_erfinv(__a); }
1537 __DEVICE__ float erfinvf(float __a) { return __nv_erfinvf(__a); }
1538 __DEVICE__ double exp(double __a) { return __nv_exp(__a); }
1539 __DEVICE__ double exp10(double __a) { return __nv_exp10(__a); }
1540 __DEVICE__ float exp10f(float __a) { return __nv_exp10f(__a); }
1541 __DEVICE__ double exp2(double __a) { return __nv_exp2(__a); }
1542 __DEVICE__ float exp2f(float __a) { return __nv_exp2f(__a); }
1543 __DEVICE__ float expf(float __a) { return __nv_expf(__a); }
1544 __DEVICE__ double expm1(double __a) { return __nv_expm1(__a); }
1545 __DEVICE__ float expm1f(float __a) { return __nv_expm1f(__a); }
1546 __DEVICE__ float fabsf(float __a) { return __nv_fabsf(__a); }
1547 __DEVICE__ double fdim(double __a, double __b) { return __nv_fdim(__a, __b); }
1548 __DEVICE__ float fdimf(float __a, float __b) { return __nv_fdimf(__a, __b); }
1549 __DEVICE__ double fdivide(double __a, double __b) { return __a / __b; }
1550 __DEVICE__ float fdividef(float __a, float __b) {
1551 #if __FAST_MATH__ && !__CUDA_PREC_DIV
1552 return __nv_fast_fdividef(__a, __b);
1557 __DEVICE__ double floor(double __f) { return __nv_floor(__f); }
1558 __DEVICE__ float floorf(float __f) { return __nv_floorf(__f); }
1559 __DEVICE__ double fma(double __a, double __b, double __c) {
1560 return __nv_fma(__a, __b, __c);
1562 __DEVICE__ float fmaf(float __a, float __b, float __c) {
1563 return __nv_fmaf(__a, __b, __c);
1565 __DEVICE__ double fmax(double __a, double __b) { return __nv_fmax(__a, __b); }
1566 __DEVICE__ float fmaxf(float __a, float __b) { return __nv_fmaxf(__a, __b); }
1567 __DEVICE__ double fmin(double __a, double __b) { return __nv_fmin(__a, __b); }
1568 __DEVICE__ float fminf(float __a, float __b) { return __nv_fminf(__a, __b); }
1569 __DEVICE__ double fmod(double __a, double __b) { return __nv_fmod(__a, __b); }
1570 __DEVICE__ float fmodf(float __a, float __b) { return __nv_fmodf(__a, __b); }
1571 __DEVICE__ double frexp(double __a, int *__b) { return __nv_frexp(__a, __b); }
1572 __DEVICE__ float frexpf(float __a, int *__b) { return __nv_frexpf(__a, __b); }
1573 __DEVICE__ double hypot(double __a, double __b) { return __nv_hypot(__a, __b); }
1574 __DEVICE__ float hypotf(float __a, float __b) { return __nv_hypotf(__a, __b); }
1575 __DEVICE__ int ilogb(double __a) { return __nv_ilogb(__a); }
1576 __DEVICE__ int ilogbf(float __a) { return __nv_ilogbf(__a); }
1577 __DEVICE__ double j0(double __a) { return __nv_j0(__a); }
1578 __DEVICE__ float j0f(float __a) { return __nv_j0f(__a); }
1579 __DEVICE__ double j1(double __a) { return __nv_j1(__a); }
1580 __DEVICE__ float j1f(float __a) { return __nv_j1f(__a); }
1581 __DEVICE__ double jn(int __n, double __a) { return __nv_jn(__n, __a); }
1582 __DEVICE__ float jnf(int __n, float __a) { return __nv_jnf(__n, __a); }
1583 #if defined(__LP64__) || defined(_WIN64)
1584 __DEVICE__ long labs(long __a) __NOEXCEPT { return __nv_llabs(__a); };
1586 __DEVICE__ long labs(long __a) __NOEXCEPT { return __nv_abs(__a); };
1588 __DEVICE__ double ldexp(double __a, int __b) { return __nv_ldexp(__a, __b); }
1589 __DEVICE__ float ldexpf(float __a, int __b) { return __nv_ldexpf(__a, __b); }
1590 __DEVICE__ double lgamma(double __a) { return __nv_lgamma(__a); }
1591 __DEVICE__ float lgammaf(float __a) { return __nv_lgammaf(__a); }
1592 __DEVICE__ long long llabs(long long __a) __NOEXCEPT { return __nv_llabs(__a); }
1593 __DEVICE__ long long llmax(long long __a, long long __b) {
1594 return __nv_llmax(__a, __b);
1596 __DEVICE__ long long llmin(long long __a, long long __b) {
1597 return __nv_llmin(__a, __b);
1599 __DEVICE__ long long llrint(double __a) { return __nv_llrint(__a); }
1600 __DEVICE__ long long llrintf(float __a) { return __nv_llrintf(__a); }
1601 __DEVICE__ long long llround(double __a) { return __nv_llround(__a); }
1602 __DEVICE__ long long llroundf(float __a) { return __nv_llroundf(__a); }
1603 __DEVICE__ double log(double __a) { return __nv_log(__a); }
1604 __DEVICE__ double log10(double __a) { return __nv_log10(__a); }
1605 __DEVICE__ float log10f(float __a) { return __nv_log10f(__a); }
1606 __DEVICE__ double log1p(double __a) { return __nv_log1p(__a); }
1607 __DEVICE__ float log1pf(float __a) { return __nv_log1pf(__a); }
1608 __DEVICE__ double log2(double __a) { return __nv_log2(__a); }
1609 __DEVICE__ float log2f(float __a) {
1610 return __FAST_OR_SLOW(__nv_fast_log2f, __nv_log2f)(__a);
1612 __DEVICE__ double logb(double __a) { return __nv_logb(__a); }
1613 __DEVICE__ float logbf(float __a) { return __nv_logbf(__a); }
1614 __DEVICE__ float logf(float __a) {
1615 return __FAST_OR_SLOW(__nv_fast_logf, __nv_logf)(__a);
1617 #if defined(__LP64__) || defined(_WIN64)
1618 __DEVICE__ long lrint(double __a) { return llrint(__a); }
1619 __DEVICE__ long lrintf(float __a) { return __float2ll_rn(__a); }
1620 __DEVICE__ long lround(double __a) { return llround(__a); }
1621 __DEVICE__ long lroundf(float __a) { return llroundf(__a); }
1623 __DEVICE__ long lrint(double __a) { return (long)rint(__a); }
1624 __DEVICE__ long lrintf(float __a) { return __float2int_rn(__a); }
1625 __DEVICE__ long lround(double __a) { return round(__a); }
1626 __DEVICE__ long lroundf(float __a) { return roundf(__a); }
1628 __DEVICE__ int max(int __a, int __b) { return __nv_max(__a, __b); }
1629 // These functions shouldn't be declared when including this header
1630 // for math function resolution purposes.
1632 __DEVICE__ void *memcpy(void *__a, const void *__b, size_t __c) {
1633 return __builtin_memcpy(__a, __b, __c);
1635 __DEVICE__ void *memset(void *__a, int __b, size_t __c) {
1636 return __builtin_memset(__a, __b, __c);
1639 __DEVICE__ int min(int __a, int __b) { return __nv_min(__a, __b); }
1640 __DEVICE__ double modf(double __a, double *__b) { return __nv_modf(__a, __b); }
1641 __DEVICE__ float modff(float __a, float *__b) { return __nv_modff(__a, __b); }
1642 __DEVICE__ double nearbyint(double __a) { return __nv_nearbyint(__a); }
1643 __DEVICE__ float nearbyintf(float __a) { return __nv_nearbyintf(__a); }
1644 __DEVICE__ double nextafter(double __a, double __b) {
1645 return __nv_nextafter(__a, __b);
1647 __DEVICE__ float nextafterf(float __a, float __b) {
1648 return __nv_nextafterf(__a, __b);
1650 __DEVICE__ double norm(int __dim, const double *__t) {
1651 return __nv_norm(__dim, __t);
1653 __DEVICE__ double norm3d(double __a, double __b, double __c) {
1654 return __nv_norm3d(__a, __b, __c);
1656 __DEVICE__ float norm3df(float __a, float __b, float __c) {
1657 return __nv_norm3df(__a, __b, __c);
1659 __DEVICE__ double norm4d(double __a, double __b, double __c, double __d) {
1660 return __nv_norm4d(__a, __b, __c, __d);
1662 __DEVICE__ float norm4df(float __a, float __b, float __c, float __d) {
1663 return __nv_norm4df(__a, __b, __c, __d);
1665 __DEVICE__ double normcdf(double __a) { return __nv_normcdf(__a); }
1666 __DEVICE__ float normcdff(float __a) { return __nv_normcdff(__a); }
1667 __DEVICE__ double normcdfinv(double __a) { return __nv_normcdfinv(__a); }
1668 __DEVICE__ float normcdfinvf(float __a) { return __nv_normcdfinvf(__a); }
1669 __DEVICE__ float normf(int __dim, const float *__t) {
1670 return __nv_normf(__dim, __t);
1672 __DEVICE__ double pow(double __a, double __b) { return __nv_pow(__a, __b); }
1673 __DEVICE__ float powf(float __a, float __b) { return __nv_powf(__a, __b); }
1674 __DEVICE__ double powi(double __a, int __b) { return __nv_powi(__a, __b); }
1675 __DEVICE__ float powif(float __a, int __b) { return __nv_powif(__a, __b); }
1676 __DEVICE__ double rcbrt(double __a) { return __nv_rcbrt(__a); }
1677 __DEVICE__ float rcbrtf(float __a) { return __nv_rcbrtf(__a); }
1678 __DEVICE__ double remainder(double __a, double __b) {
1679 return __nv_remainder(__a, __b);
1681 __DEVICE__ float remainderf(float __a, float __b) {
1682 return __nv_remainderf(__a, __b);
1684 __DEVICE__ double remquo(double __a, double __b, int *__c) {
1685 return __nv_remquo(__a, __b, __c);
1687 __DEVICE__ float remquof(float __a, float __b, int *__c) {
1688 return __nv_remquof(__a, __b, __c);
1690 __DEVICE__ double rhypot(double __a, double __b) {
1691 return __nv_rhypot(__a, __b);
1693 __DEVICE__ float rhypotf(float __a, float __b) {
1694 return __nv_rhypotf(__a, __b);
1696 __DEVICE__ double rint(double __a) { return __nv_rint(__a); }
1697 __DEVICE__ float rintf(float __a) { return __nv_rintf(__a); }
1698 __DEVICE__ double rnorm(int __a, const double *__b) {
1699 return __nv_rnorm(__a, __b);
1701 __DEVICE__ double rnorm3d(double __a, double __b, double __c) {
1702 return __nv_rnorm3d(__a, __b, __c);
1704 __DEVICE__ float rnorm3df(float __a, float __b, float __c) {
1705 return __nv_rnorm3df(__a, __b, __c);
1707 __DEVICE__ double rnorm4d(double __a, double __b, double __c, double __d) {
1708 return __nv_rnorm4d(__a, __b, __c, __d);
1710 __DEVICE__ float rnorm4df(float __a, float __b, float __c, float __d) {
1711 return __nv_rnorm4df(__a, __b, __c, __d);
1713 __DEVICE__ float rnormf(int __dim, const float *__t) {
1714 return __nv_rnormf(__dim, __t);
1716 __DEVICE__ double round(double __a) { return __nv_round(__a); }
1717 __DEVICE__ float roundf(float __a) { return __nv_roundf(__a); }
1718 __DEVICE__ double rsqrt(double __a) { return __nv_rsqrt(__a); }
1719 __DEVICE__ float rsqrtf(float __a) { return __nv_rsqrtf(__a); }
1720 __DEVICE__ double scalbn(double __a, int __b) { return __nv_scalbn(__a, __b); }
1721 __DEVICE__ float scalbnf(float __a, int __b) { return __nv_scalbnf(__a, __b); }
1722 // TODO: remove once variant is supported
1724 __DEVICE__ double scalbln(double __a, long __b) {
1726 return __a > 0 ? HUGE_VAL : -HUGE_VAL;
1728 return __a > 0 ? 0.0 : -0.0;
1729 return scalbn(__a, (int)__b);
1731 __DEVICE__ float scalblnf(float __a, long __b) {
1733 return __a > 0 ? HUGE_VALF : -HUGE_VALF;
1735 return __a > 0 ? 0.f : -0.f;
1736 return scalbnf(__a, (int)__b);
1739 __DEVICE__ double sin(double __a) { return __nv_sin(__a); }
1740 __DEVICE__ void sincos(double __a, double *__s, double *__c) {
1741 return __nv_sincos(__a, __s, __c);
1743 __DEVICE__ void sincosf(float __a, float *__s, float *__c) {
1744 return __FAST_OR_SLOW(__nv_fast_sincosf, __nv_sincosf)(__a, __s, __c);
1746 __DEVICE__ void sincospi(double __a, double *__s, double *__c) {
1747 return __nv_sincospi(__a, __s, __c);
1749 __DEVICE__ void sincospif(float __a, float *__s, float *__c) {
1750 return __nv_sincospif(__a, __s, __c);
1752 __DEVICE__ float sinf(float __a) {
1753 return __FAST_OR_SLOW(__nv_fast_sinf, __nv_sinf)(__a);
1755 __DEVICE__ double sinh(double __a) { return __nv_sinh(__a); }
1756 __DEVICE__ float sinhf(float __a) { return __nv_sinhf(__a); }
1757 __DEVICE__ double sinpi(double __a) { return __nv_sinpi(__a); }
1758 __DEVICE__ float sinpif(float __a) { return __nv_sinpif(__a); }
1759 __DEVICE__ double sqrt(double __a) { return __nv_sqrt(__a); }
1760 __DEVICE__ float sqrtf(float __a) { return __nv_sqrtf(__a); }
1761 __DEVICE__ double tan(double __a) { return __nv_tan(__a); }
1762 __DEVICE__ float tanf(float __a) { return __nv_tanf(__a); }
1763 __DEVICE__ double tanh(double __a) { return __nv_tanh(__a); }
1764 __DEVICE__ float tanhf(float __a) { return __nv_tanhf(__a); }
1765 __DEVICE__ double tgamma(double __a) { return __nv_tgamma(__a); }
1766 __DEVICE__ float tgammaf(float __a) { return __nv_tgammaf(__a); }
1767 __DEVICE__ double trunc(double __a) { return __nv_trunc(__a); }
1768 __DEVICE__ float truncf(float __a) { return __nv_truncf(__a); }
1769 __DEVICE__ unsigned long long ullmax(unsigned long long __a,
1770 unsigned long long __b) {
1771 return __nv_ullmax(__a, __b);
1773 __DEVICE__ unsigned long long ullmin(unsigned long long __a,
1774 unsigned long long __b) {
1775 return __nv_ullmin(__a, __b);
1777 __DEVICE__ unsigned int umax(unsigned int __a, unsigned int __b) {
1778 return __nv_umax(__a, __b);
1780 __DEVICE__ unsigned int umin(unsigned int __a, unsigned int __b) {
1781 return __nv_umin(__a, __b);
1783 __DEVICE__ double y0(double __a) { return __nv_y0(__a); }
1784 __DEVICE__ float y0f(float __a) { return __nv_y0f(__a); }
1785 __DEVICE__ double y1(double __a) { return __nv_y1(__a); }
1786 __DEVICE__ float y1f(float __a) { return __nv_y1f(__a); }
1787 __DEVICE__ double yn(int __a, double __b) { return __nv_yn(__a, __b); }
1788 __DEVICE__ float ynf(int __a, float __b) { return __nv_ynf(__a, __b); }
1791 #pragma pop_macro("__DEVICE__")
1792 #pragma pop_macro("__FAST_OR_SLOW")
1793 #endif // __CLANG_CUDA_DEVICE_FUNCTIONS_H__