contrib/llvm-project/clang/include/clang/Basic/BuiltinsNVPTX.def

   1 //===--- BuiltinsPTX.def - PTX Builtin function database ----*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file defines the PTX-specific builtin function database.  Users of
  10 // this file must define the BUILTIN macro to make use of this information.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 // The format of this database matches clang/Basic/Builtins.def.
  15
  16 #if defined(BUILTIN) && !defined(TARGET_BUILTIN)
  17 #   define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS)
  18 #endif
  19
  20 #pragma push_macro("SM_70")
  21 #pragma push_macro("SM_72")
  22 #pragma push_macro("SM_75")
  23 #define SM_75 "sm_75"
  24 #define SM_72 "sm_72|" SM_75
  25 #define SM_70 "sm_70|" SM_72
  26
  27 #pragma push_macro("SM_60")
  28 #define SM_60 "sm_60|sm_61|sm_62|" SM_70
  29
  30 #pragma push_macro("PTX60")
  31 #pragma push_macro("PTX61")
  32 #pragma push_macro("PTX63")
  33 #pragma push_macro("PTX64")
  34 #define PTX64 "ptx64"
  35 #define PTX63 "ptx63|" PTX64
  36 #define PTX61 "ptx61|" PTX63
  37 #define PTX60 "ptx60|" PTX61
  38
  39 #pragma push_macro("AND")
  40 #define AND(a, b) a "," b
  41
  42 // Special Registers
  43
  44 BUILTIN(__nvvm_read_ptx_sreg_tid_x, "i", "nc")
  45 BUILTIN(__nvvm_read_ptx_sreg_tid_y, "i", "nc")
  46 BUILTIN(__nvvm_read_ptx_sreg_tid_z, "i", "nc")
  47 BUILTIN(__nvvm_read_ptx_sreg_tid_w, "i", "nc")
  48
  49 BUILTIN(__nvvm_read_ptx_sreg_ntid_x, "i", "nc")
  50 BUILTIN(__nvvm_read_ptx_sreg_ntid_y, "i", "nc")
  51 BUILTIN(__nvvm_read_ptx_sreg_ntid_z, "i", "nc")
  52 BUILTIN(__nvvm_read_ptx_sreg_ntid_w, "i", "nc")
  53
  54 BUILTIN(__nvvm_read_ptx_sreg_ctaid_x, "i", "nc")
  55 BUILTIN(__nvvm_read_ptx_sreg_ctaid_y, "i", "nc")
  56 BUILTIN(__nvvm_read_ptx_sreg_ctaid_z, "i", "nc")
  57 BUILTIN(__nvvm_read_ptx_sreg_ctaid_w, "i", "nc")
  58
  59 BUILTIN(__nvvm_read_ptx_sreg_nctaid_x, "i", "nc")
  60 BUILTIN(__nvvm_read_ptx_sreg_nctaid_y, "i", "nc")
  61 BUILTIN(__nvvm_read_ptx_sreg_nctaid_z, "i", "nc")
  62 BUILTIN(__nvvm_read_ptx_sreg_nctaid_w, "i", "nc")
  63
  64 BUILTIN(__nvvm_read_ptx_sreg_laneid, "i", "nc")
  65 BUILTIN(__nvvm_read_ptx_sreg_warpid, "i", "nc")
  66 BUILTIN(__nvvm_read_ptx_sreg_nwarpid, "i", "nc")
  67
  68 BUILTIN(__nvvm_read_ptx_sreg_smid, "i", "nc")
  69 BUILTIN(__nvvm_read_ptx_sreg_nsmid, "i", "nc")
  70 BUILTIN(__nvvm_read_ptx_sreg_gridid, "i", "nc")
  71
  72 BUILTIN(__nvvm_read_ptx_sreg_lanemask_eq, "i", "nc")
  73 BUILTIN(__nvvm_read_ptx_sreg_lanemask_le, "i", "nc")
  74 BUILTIN(__nvvm_read_ptx_sreg_lanemask_lt, "i", "nc")
  75 BUILTIN(__nvvm_read_ptx_sreg_lanemask_ge, "i", "nc")
  76 BUILTIN(__nvvm_read_ptx_sreg_lanemask_gt, "i", "nc")
  77
  78 BUILTIN(__nvvm_read_ptx_sreg_clock, "i", "n")
  79 BUILTIN(__nvvm_read_ptx_sreg_clock64, "LLi", "n")
  80
  81 BUILTIN(__nvvm_read_ptx_sreg_pm0, "i", "n")
  82 BUILTIN(__nvvm_read_ptx_sreg_pm1, "i", "n")
  83 BUILTIN(__nvvm_read_ptx_sreg_pm2, "i", "n")
  84 BUILTIN(__nvvm_read_ptx_sreg_pm3, "i", "n")
  85
  86 // MISC
  87
  88 BUILTIN(__nvvm_prmt, "UiUiUiUi", "")
  89
  90 // Min Max
  91
  92 BUILTIN(__nvvm_fmax_ftz_f, "fff",  "")
  93 BUILTIN(__nvvm_fmax_f, "fff",  "")
  94 BUILTIN(__nvvm_fmin_ftz_f, "fff",  "")
  95 BUILTIN(__nvvm_fmin_f, "fff",  "")
  96
  97 BUILTIN(__nvvm_fmax_d, "ddd", "")
  98 BUILTIN(__nvvm_fmin_d, "ddd", "")
  99
 100 // Multiplication
 101
 102 BUILTIN(__nvvm_mulhi_i, "iii", "")
 103 BUILTIN(__nvvm_mulhi_ui, "UiUiUi", "")
 104 BUILTIN(__nvvm_mulhi_ll, "LLiLLiLLi", "")
 105 BUILTIN(__nvvm_mulhi_ull, "ULLiULLiULLi", "")
 106
 107 BUILTIN(__nvvm_mul_rn_ftz_f,  "fff", "")
 108 BUILTIN(__nvvm_mul_rn_f,  "fff", "")
 109 BUILTIN(__nvvm_mul_rz_ftz_f,  "fff", "")
 110 BUILTIN(__nvvm_mul_rz_f,  "fff", "")
 111 BUILTIN(__nvvm_mul_rm_ftz_f,  "fff", "")
 112 BUILTIN(__nvvm_mul_rm_f,  "fff", "")
 113 BUILTIN(__nvvm_mul_rp_ftz_f,  "fff", "")
 114 BUILTIN(__nvvm_mul_rp_f,  "fff", "")
 115
 116 BUILTIN(__nvvm_mul_rn_d,  "ddd", "")
 117 BUILTIN(__nvvm_mul_rz_d,  "ddd", "")
 118 BUILTIN(__nvvm_mul_rm_d,  "ddd", "")
 119 BUILTIN(__nvvm_mul_rp_d,  "ddd", "")
 120
 121 BUILTIN(__nvvm_mul24_i,  "iii", "")
 122 BUILTIN(__nvvm_mul24_ui,  "UiUiUi", "")
 123
 124 // Div
 125
 126 BUILTIN(__nvvm_div_approx_ftz_f,  "fff", "")
 127 BUILTIN(__nvvm_div_approx_f,  "fff", "")
 128
 129 BUILTIN(__nvvm_div_rn_ftz_f,  "fff", "")
 130 BUILTIN(__nvvm_div_rn_f,  "fff", "")
 131 BUILTIN(__nvvm_div_rz_ftz_f,  "fff", "")
 132 BUILTIN(__nvvm_div_rz_f,  "fff", "")
 133 BUILTIN(__nvvm_div_rm_ftz_f,  "fff", "")
 134 BUILTIN(__nvvm_div_rm_f,  "fff", "")
 135 BUILTIN(__nvvm_div_rp_ftz_f,  "fff", "")
 136 BUILTIN(__nvvm_div_rp_f,  "fff", "")
 137
 138 BUILTIN(__nvvm_div_rn_d,  "ddd", "")
 139 BUILTIN(__nvvm_div_rz_d,  "ddd", "")
 140 BUILTIN(__nvvm_div_rm_d,  "ddd", "")
 141 BUILTIN(__nvvm_div_rp_d,  "ddd", "")
 142
 143 // Sad
 144
 145 BUILTIN(__nvvm_sad_i, "iiii", "")
 146 BUILTIN(__nvvm_sad_ui, "UiUiUiUi", "")
 147
 148 // Floor, Ceil
 149
 150 BUILTIN(__nvvm_floor_ftz_f, "ff", "")
 151 BUILTIN(__nvvm_floor_f, "ff", "")
 152 BUILTIN(__nvvm_floor_d, "dd", "")
 153
 154 BUILTIN(__nvvm_ceil_ftz_f, "ff", "")
 155 BUILTIN(__nvvm_ceil_f, "ff", "")
 156 BUILTIN(__nvvm_ceil_d, "dd", "")
 157
 158 // Abs
 159
 160 BUILTIN(__nvvm_fabs_ftz_f, "ff", "")
 161 BUILTIN(__nvvm_fabs_f, "ff", "")
 162 BUILTIN(__nvvm_fabs_d, "dd", "")
 163
 164 // Round
 165
 166 BUILTIN(__nvvm_round_ftz_f, "ff", "")
 167 BUILTIN(__nvvm_round_f, "ff", "")
 168 BUILTIN(__nvvm_round_d, "dd", "")
 169
 170 // Trunc
 171
 172 BUILTIN(__nvvm_trunc_ftz_f, "ff", "")
 173 BUILTIN(__nvvm_trunc_f, "ff", "")
 174 BUILTIN(__nvvm_trunc_d, "dd", "")
 175
 176 // Saturate
 177
 178 BUILTIN(__nvvm_saturate_ftz_f, "ff", "")
 179 BUILTIN(__nvvm_saturate_f, "ff", "")
 180 BUILTIN(__nvvm_saturate_d, "dd", "")
 181
 182 // Exp2, Log2
 183
 184 BUILTIN(__nvvm_ex2_approx_ftz_f, "ff", "")
 185 BUILTIN(__nvvm_ex2_approx_f, "ff", "")
 186 BUILTIN(__nvvm_ex2_approx_d, "dd", "")
 187
 188 BUILTIN(__nvvm_lg2_approx_ftz_f, "ff", "")
 189 BUILTIN(__nvvm_lg2_approx_f, "ff", "")
 190 BUILTIN(__nvvm_lg2_approx_d, "dd", "")
 191
 192 // Sin, Cos
 193
 194 BUILTIN(__nvvm_sin_approx_ftz_f, "ff", "")
 195 BUILTIN(__nvvm_sin_approx_f, "ff", "")
 196
 197 BUILTIN(__nvvm_cos_approx_ftz_f, "ff", "")
 198 BUILTIN(__nvvm_cos_approx_f, "ff", "")
 199
 200 // Fma
 201
 202 BUILTIN(__nvvm_fma_rn_ftz_f, "ffff", "")
 203 BUILTIN(__nvvm_fma_rn_f, "ffff", "")
 204 BUILTIN(__nvvm_fma_rz_ftz_f, "ffff", "")
 205 BUILTIN(__nvvm_fma_rz_f, "ffff", "")
 206 BUILTIN(__nvvm_fma_rm_ftz_f, "ffff", "")
 207 BUILTIN(__nvvm_fma_rm_f, "ffff", "")
 208 BUILTIN(__nvvm_fma_rp_ftz_f, "ffff", "")
 209 BUILTIN(__nvvm_fma_rp_f, "ffff", "")
 210 BUILTIN(__nvvm_fma_rn_d, "dddd", "")
 211 BUILTIN(__nvvm_fma_rz_d, "dddd", "")
 212 BUILTIN(__nvvm_fma_rm_d, "dddd", "")
 213 BUILTIN(__nvvm_fma_rp_d, "dddd", "")
 214
 215 // Rcp
 216
 217 BUILTIN(__nvvm_rcp_rn_ftz_f, "ff", "")
 218 BUILTIN(__nvvm_rcp_rn_f, "ff", "")
 219 BUILTIN(__nvvm_rcp_rz_ftz_f, "ff", "")
 220 BUILTIN(__nvvm_rcp_rz_f, "ff", "")
 221 BUILTIN(__nvvm_rcp_rm_ftz_f, "ff", "")
 222 BUILTIN(__nvvm_rcp_rm_f, "ff", "")
 223 BUILTIN(__nvvm_rcp_rp_ftz_f, "ff", "")
 224 BUILTIN(__nvvm_rcp_rp_f, "ff", "")
 225
 226 BUILTIN(__nvvm_rcp_rn_d, "dd", "")
 227 BUILTIN(__nvvm_rcp_rz_d, "dd", "")
 228 BUILTIN(__nvvm_rcp_rm_d, "dd", "")
 229 BUILTIN(__nvvm_rcp_rp_d, "dd", "")
 230 BUILTIN(__nvvm_rcp_approx_ftz_d, "dd", "")
 231
 232 // Sqrt
 233
 234 BUILTIN(__nvvm_sqrt_rn_ftz_f, "ff", "")
 235 BUILTIN(__nvvm_sqrt_rn_f, "ff", "")
 236 BUILTIN(__nvvm_sqrt_rz_ftz_f, "ff", "")
 237 BUILTIN(__nvvm_sqrt_rz_f, "ff", "")
 238 BUILTIN(__nvvm_sqrt_rm_ftz_f, "ff", "")
 239 BUILTIN(__nvvm_sqrt_rm_f, "ff", "")
 240 BUILTIN(__nvvm_sqrt_rp_ftz_f, "ff", "")
 241 BUILTIN(__nvvm_sqrt_rp_f, "ff", "")
 242 BUILTIN(__nvvm_sqrt_approx_ftz_f, "ff", "")
 243 BUILTIN(__nvvm_sqrt_approx_f, "ff", "")
 244
 245 BUILTIN(__nvvm_sqrt_rn_d, "dd", "")
 246 BUILTIN(__nvvm_sqrt_rz_d, "dd", "")
 247 BUILTIN(__nvvm_sqrt_rm_d, "dd", "")
 248 BUILTIN(__nvvm_sqrt_rp_d, "dd", "")
 249
 250 // Rsqrt
 251
 252 BUILTIN(__nvvm_rsqrt_approx_ftz_f, "ff", "")
 253 BUILTIN(__nvvm_rsqrt_approx_f, "ff", "")
 254 BUILTIN(__nvvm_rsqrt_approx_d, "dd", "")
 255
 256 // Add
 257
 258 BUILTIN(__nvvm_add_rn_ftz_f, "fff", "")
 259 BUILTIN(__nvvm_add_rn_f, "fff", "")
 260 BUILTIN(__nvvm_add_rz_ftz_f, "fff", "")
 261 BUILTIN(__nvvm_add_rz_f, "fff", "")
 262 BUILTIN(__nvvm_add_rm_ftz_f, "fff", "")
 263 BUILTIN(__nvvm_add_rm_f, "fff", "")
 264 BUILTIN(__nvvm_add_rp_ftz_f, "fff", "")
 265 BUILTIN(__nvvm_add_rp_f, "fff", "")
 266
 267 BUILTIN(__nvvm_add_rn_d, "ddd", "")
 268 BUILTIN(__nvvm_add_rz_d, "ddd", "")
 269 BUILTIN(__nvvm_add_rm_d, "ddd", "")
 270 BUILTIN(__nvvm_add_rp_d, "ddd", "")
 271
 272 // Convert
 273
 274 BUILTIN(__nvvm_d2f_rn_ftz, "fd", "")
 275 BUILTIN(__nvvm_d2f_rn, "fd", "")
 276 BUILTIN(__nvvm_d2f_rz_ftz, "fd", "")
 277 BUILTIN(__nvvm_d2f_rz, "fd", "")
 278 BUILTIN(__nvvm_d2f_rm_ftz, "fd", "")
 279 BUILTIN(__nvvm_d2f_rm, "fd", "")
 280 BUILTIN(__nvvm_d2f_rp_ftz, "fd", "")
 281 BUILTIN(__nvvm_d2f_rp, "fd", "")
 282
 283 BUILTIN(__nvvm_d2i_rn, "id", "")
 284 BUILTIN(__nvvm_d2i_rz, "id", "")
 285 BUILTIN(__nvvm_d2i_rm, "id", "")
 286 BUILTIN(__nvvm_d2i_rp, "id", "")
 287
 288 BUILTIN(__nvvm_d2ui_rn, "Uid", "")
 289 BUILTIN(__nvvm_d2ui_rz, "Uid", "")
 290 BUILTIN(__nvvm_d2ui_rm, "Uid", "")
 291 BUILTIN(__nvvm_d2ui_rp, "Uid", "")
 292
 293 BUILTIN(__nvvm_i2d_rn, "di", "")
 294 BUILTIN(__nvvm_i2d_rz, "di", "")
 295 BUILTIN(__nvvm_i2d_rm, "di", "")
 296 BUILTIN(__nvvm_i2d_rp, "di", "")
 297
 298 BUILTIN(__nvvm_ui2d_rn, "dUi", "")
 299 BUILTIN(__nvvm_ui2d_rz, "dUi", "")
 300 BUILTIN(__nvvm_ui2d_rm, "dUi", "")
 301 BUILTIN(__nvvm_ui2d_rp, "dUi", "")
 302
 303 BUILTIN(__nvvm_f2i_rn_ftz, "if", "")
 304 BUILTIN(__nvvm_f2i_rn, "if", "")
 305 BUILTIN(__nvvm_f2i_rz_ftz, "if", "")
 306 BUILTIN(__nvvm_f2i_rz, "if", "")
 307 BUILTIN(__nvvm_f2i_rm_ftz, "if", "")
 308 BUILTIN(__nvvm_f2i_rm, "if", "")
 309 BUILTIN(__nvvm_f2i_rp_ftz, "if", "")
 310 BUILTIN(__nvvm_f2i_rp, "if", "")
 311
 312 BUILTIN(__nvvm_f2ui_rn_ftz, "Uif", "")
 313 BUILTIN(__nvvm_f2ui_rn, "Uif", "")
 314 BUILTIN(__nvvm_f2ui_rz_ftz, "Uif", "")
 315 BUILTIN(__nvvm_f2ui_rz, "Uif", "")
 316 BUILTIN(__nvvm_f2ui_rm_ftz, "Uif", "")
 317 BUILTIN(__nvvm_f2ui_rm, "Uif", "")
 318 BUILTIN(__nvvm_f2ui_rp_ftz, "Uif", "")
 319 BUILTIN(__nvvm_f2ui_rp, "Uif", "")
 320
 321 BUILTIN(__nvvm_i2f_rn, "fi", "")
 322 BUILTIN(__nvvm_i2f_rz, "fi", "")
 323 BUILTIN(__nvvm_i2f_rm, "fi", "")
 324 BUILTIN(__nvvm_i2f_rp, "fi", "")
 325
 326 BUILTIN(__nvvm_ui2f_rn, "fUi", "")
 327 BUILTIN(__nvvm_ui2f_rz, "fUi", "")
 328 BUILTIN(__nvvm_ui2f_rm, "fUi", "")
 329 BUILTIN(__nvvm_ui2f_rp, "fUi", "")
 330
 331 BUILTIN(__nvvm_lohi_i2d, "dii", "")
 332
 333 BUILTIN(__nvvm_d2i_lo, "id", "")
 334 BUILTIN(__nvvm_d2i_hi, "id", "")
 335
 336 BUILTIN(__nvvm_f2ll_rn_ftz, "LLif", "")
 337 BUILTIN(__nvvm_f2ll_rn, "LLif", "")
 338 BUILTIN(__nvvm_f2ll_rz_ftz, "LLif", "")
 339 BUILTIN(__nvvm_f2ll_rz, "LLif", "")
 340 BUILTIN(__nvvm_f2ll_rm_ftz, "LLif", "")
 341 BUILTIN(__nvvm_f2ll_rm, "LLif", "")
 342 BUILTIN(__nvvm_f2ll_rp_ftz, "LLif", "")
 343 BUILTIN(__nvvm_f2ll_rp, "LLif", "")
 344
 345 BUILTIN(__nvvm_f2ull_rn_ftz, "ULLif", "")
 346 BUILTIN(__nvvm_f2ull_rn, "ULLif", "")
 347 BUILTIN(__nvvm_f2ull_rz_ftz, "ULLif", "")
 348 BUILTIN(__nvvm_f2ull_rz, "ULLif", "")
 349 BUILTIN(__nvvm_f2ull_rm_ftz, "ULLif", "")
 350 BUILTIN(__nvvm_f2ull_rm, "ULLif", "")
 351 BUILTIN(__nvvm_f2ull_rp_ftz, "ULLif", "")
 352 BUILTIN(__nvvm_f2ull_rp, "ULLif", "")
 353
 354 BUILTIN(__nvvm_d2ll_rn, "LLid", "")
 355 BUILTIN(__nvvm_d2ll_rz, "LLid", "")
 356 BUILTIN(__nvvm_d2ll_rm, "LLid", "")
 357 BUILTIN(__nvvm_d2ll_rp, "LLid", "")
 358
 359 BUILTIN(__nvvm_d2ull_rn, "ULLid", "")
 360 BUILTIN(__nvvm_d2ull_rz, "ULLid", "")
 361 BUILTIN(__nvvm_d2ull_rm, "ULLid", "")
 362 BUILTIN(__nvvm_d2ull_rp, "ULLid", "")
 363
 364 BUILTIN(__nvvm_ll2f_rn, "fLLi", "")
 365 BUILTIN(__nvvm_ll2f_rz, "fLLi", "")
 366 BUILTIN(__nvvm_ll2f_rm, "fLLi", "")
 367 BUILTIN(__nvvm_ll2f_rp, "fLLi", "")
 368
 369 BUILTIN(__nvvm_ull2f_rn, "fULLi", "")
 370 BUILTIN(__nvvm_ull2f_rz, "fULLi", "")
 371 BUILTIN(__nvvm_ull2f_rm, "fULLi", "")
 372 BUILTIN(__nvvm_ull2f_rp, "fULLi", "")
 373
 374 BUILTIN(__nvvm_ll2d_rn, "dLLi", "")
 375 BUILTIN(__nvvm_ll2d_rz, "dLLi", "")
 376 BUILTIN(__nvvm_ll2d_rm, "dLLi", "")
 377 BUILTIN(__nvvm_ll2d_rp, "dLLi", "")
 378
 379 BUILTIN(__nvvm_ull2d_rn, "dULLi", "")
 380 BUILTIN(__nvvm_ull2d_rz, "dULLi", "")
 381 BUILTIN(__nvvm_ull2d_rm, "dULLi", "")
 382 BUILTIN(__nvvm_ull2d_rp, "dULLi", "")
 383
 384 BUILTIN(__nvvm_f2h_rn_ftz, "Usf", "")
 385 BUILTIN(__nvvm_f2h_rn, "Usf", "")
 386
 387 // Bitcast
 388
 389 BUILTIN(__nvvm_bitcast_f2i, "if", "")
 390 BUILTIN(__nvvm_bitcast_i2f, "fi", "")
 391
 392 BUILTIN(__nvvm_bitcast_ll2d, "dLLi", "")
 393 BUILTIN(__nvvm_bitcast_d2ll, "LLid", "")
 394
 395 // FNS
 396 TARGET_BUILTIN(__nvvm_fns, "UiUiUii", "n", PTX60)
 397
 398 // Sync
 399
 400 BUILTIN(__syncthreads, "v", "")
 401 BUILTIN(__nvvm_bar0_popc, "ii", "")
 402 BUILTIN(__nvvm_bar0_and, "ii", "")
 403 BUILTIN(__nvvm_bar0_or, "ii", "")
 404 BUILTIN(__nvvm_bar_sync, "vi", "n")
 405 TARGET_BUILTIN(__nvvm_bar_warp_sync, "vUi", "n", PTX60)
 406 TARGET_BUILTIN(__nvvm_barrier_sync, "vUi", "n", PTX60)
 407 TARGET_BUILTIN(__nvvm_barrier_sync_cnt, "vUiUi", "n", PTX60)
 408
 409 // Shuffle
 410
 411 BUILTIN(__nvvm_shfl_down_i32, "iiii", "")
 412 BUILTIN(__nvvm_shfl_down_f32, "ffii", "")
 413 BUILTIN(__nvvm_shfl_up_i32, "iiii", "")
 414 BUILTIN(__nvvm_shfl_up_f32, "ffii", "")
 415 BUILTIN(__nvvm_shfl_bfly_i32, "iiii", "")
 416 BUILTIN(__nvvm_shfl_bfly_f32, "ffii", "")
 417 BUILTIN(__nvvm_shfl_idx_i32, "iiii", "")
 418 BUILTIN(__nvvm_shfl_idx_f32, "ffii", "")
 419
 420 TARGET_BUILTIN(__nvvm_shfl_sync_down_i32, "iUiiii", "", PTX60)
 421 TARGET_BUILTIN(__nvvm_shfl_sync_down_f32, "fUifii", "", PTX60)
 422 TARGET_BUILTIN(__nvvm_shfl_sync_up_i32, "iUiiii", "", PTX60)
 423 TARGET_BUILTIN(__nvvm_shfl_sync_up_f32, "fUifii", "", PTX60)
 424 TARGET_BUILTIN(__nvvm_shfl_sync_bfly_i32, "iUiiii", "", PTX60)
 425 TARGET_BUILTIN(__nvvm_shfl_sync_bfly_f32, "fUifii", "", PTX60)
 426 TARGET_BUILTIN(__nvvm_shfl_sync_idx_i32, "iUiiii", "", PTX60)
 427 TARGET_BUILTIN(__nvvm_shfl_sync_idx_f32, "fUifii", "", PTX60)
 428
 429 // Vote
 430 BUILTIN(__nvvm_vote_all, "bb", "")
 431 BUILTIN(__nvvm_vote_any, "bb", "")
 432 BUILTIN(__nvvm_vote_uni, "bb", "")
 433 BUILTIN(__nvvm_vote_ballot, "Uib", "")
 434
 435 TARGET_BUILTIN(__nvvm_vote_all_sync, "bUib", "", PTX60)
 436 TARGET_BUILTIN(__nvvm_vote_any_sync, "bUib", "", PTX60)
 437 TARGET_BUILTIN(__nvvm_vote_uni_sync, "bUib", "", PTX60)
 438 TARGET_BUILTIN(__nvvm_vote_ballot_sync, "UiUib", "", PTX60)
 439
 440 // Match
 441 TARGET_BUILTIN(__nvvm_match_any_sync_i32, "UiUiUi", "", PTX60)
 442 TARGET_BUILTIN(__nvvm_match_any_sync_i64, "WiUiWi", "", PTX60)
 443 // These return a pair {value, predicate}, which requires custom lowering.
 444 TARGET_BUILTIN(__nvvm_match_all_sync_i32p, "UiUiUii*", "", PTX60)
 445 TARGET_BUILTIN(__nvvm_match_all_sync_i64p, "WiUiWii*", "", PTX60)
 446
 447 // Membar
 448
 449 BUILTIN(__nvvm_membar_cta, "v", "")
 450 BUILTIN(__nvvm_membar_gl, "v", "")
 451 BUILTIN(__nvvm_membar_sys, "v", "")
 452
 453 // Memcpy, Memset
 454
 455 BUILTIN(__nvvm_memcpy, "vUc*Uc*zi","")
 456 BUILTIN(__nvvm_memset, "vUc*Uczi","")
 457
 458 // Image
 459
 460 BUILTIN(__builtin_ptx_read_image2Dfi_, "V4fiiii", "")
 461 BUILTIN(__builtin_ptx_read_image2Dff_, "V4fiiff", "")
 462 BUILTIN(__builtin_ptx_read_image2Dii_, "V4iiiii", "")
 463 BUILTIN(__builtin_ptx_read_image2Dif_, "V4iiiff", "")
 464
 465 BUILTIN(__builtin_ptx_read_image3Dfi_, "V4fiiiiii", "")
 466 BUILTIN(__builtin_ptx_read_image3Dff_, "V4fiiffff", "")
 467 BUILTIN(__builtin_ptx_read_image3Dii_, "V4iiiiiii", "")
 468 BUILTIN(__builtin_ptx_read_image3Dif_, "V4iiiffff", "")
 469
 470 BUILTIN(__builtin_ptx_write_image2Df_, "viiiffff", "")
 471 BUILTIN(__builtin_ptx_write_image2Di_, "viiiiiii", "")
 472 BUILTIN(__builtin_ptx_write_image2Dui_, "viiiUiUiUiUi", "")
 473 BUILTIN(__builtin_ptx_get_image_depthi_, "ii", "")
 474 BUILTIN(__builtin_ptx_get_image_heighti_, "ii", "")
 475 BUILTIN(__builtin_ptx_get_image_widthi_, "ii", "")
 476 BUILTIN(__builtin_ptx_get_image_channel_data_typei_, "ii", "")
 477 BUILTIN(__builtin_ptx_get_image_channel_orderi_, "ii", "")
 478
 479 // Atomic
 480 //
 481 // We need the atom intrinsics because
 482 // - they are used in converging analysis
 483 // - they are used in address space analysis and optimization
 484 // So it does not hurt to expose them as builtins.
 485 //
 486 BUILTIN(__nvvm_atom_add_gen_i, "iiD*i", "n")
 487 TARGET_BUILTIN(__nvvm_atom_cta_add_gen_i, "iiD*i", "n", SM_60)
 488 TARGET_BUILTIN(__nvvm_atom_sys_add_gen_i, "iiD*i", "n", SM_60)
 489 BUILTIN(__nvvm_atom_add_gen_l, "LiLiD*Li", "n")
 490 TARGET_BUILTIN(__nvvm_atom_cta_add_gen_l, "LiLiD*Li", "n", SM_60)
 491 TARGET_BUILTIN(__nvvm_atom_sys_add_gen_l, "LiLiD*Li", "n", SM_60)
 492 BUILTIN(__nvvm_atom_add_gen_ll, "LLiLLiD*LLi", "n")
 493 TARGET_BUILTIN(__nvvm_atom_cta_add_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 494 TARGET_BUILTIN(__nvvm_atom_sys_add_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 495 BUILTIN(__nvvm_atom_add_gen_f, "ffD*f", "n")
 496 TARGET_BUILTIN(__nvvm_atom_cta_add_gen_f, "ffD*f", "n", SM_60)
 497 TARGET_BUILTIN(__nvvm_atom_sys_add_gen_f, "ffD*f", "n", SM_60)
 498 TARGET_BUILTIN(__nvvm_atom_add_gen_d, "ddD*d", "n", SM_60)
 499 TARGET_BUILTIN(__nvvm_atom_cta_add_gen_d, "ddD*d", "n", SM_60)
 500 TARGET_BUILTIN(__nvvm_atom_sys_add_gen_d, "ddD*d", "n", SM_60)
 501
 502 BUILTIN(__nvvm_atom_sub_gen_i, "iiD*i", "n")
 503 BUILTIN(__nvvm_atom_sub_gen_l, "LiLiD*Li", "n")
 504 BUILTIN(__nvvm_atom_sub_gen_ll, "LLiLLiD*LLi", "n")
 505
 506 BUILTIN(__nvvm_atom_xchg_gen_i, "iiD*i", "n")
 507 TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_i, "iiD*i", "n", SM_60)
 508 TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_i, "iiD*i", "n", SM_60)
 509 BUILTIN(__nvvm_atom_xchg_gen_l, "LiLiD*Li", "n")
 510 TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_l, "LiLiD*Li", "n", SM_60)
 511 TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_l, "LiLiD*Li", "n", SM_60)
 512 BUILTIN(__nvvm_atom_xchg_gen_ll, "LLiLLiD*LLi", "n")
 513 TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 514 TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 515
 516 BUILTIN(__nvvm_atom_max_gen_i, "iiD*i", "n")
 517 TARGET_BUILTIN(__nvvm_atom_cta_max_gen_i, "iiD*i", "n", SM_60)
 518 TARGET_BUILTIN(__nvvm_atom_sys_max_gen_i, "iiD*i", "n", SM_60)
 519 BUILTIN(__nvvm_atom_max_gen_ui, "UiUiD*Ui", "n")
 520 TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ui, "UiUiD*Ui", "n", SM_60)
 521 TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ui, "UiUiD*Ui", "n", SM_60)
 522 BUILTIN(__nvvm_atom_max_gen_l, "LiLiD*Li", "n")
 523 TARGET_BUILTIN(__nvvm_atom_cta_max_gen_l, "LiLiD*Li", "n", SM_60)
 524 TARGET_BUILTIN(__nvvm_atom_sys_max_gen_l, "LiLiD*Li", "n", SM_60)
 525 BUILTIN(__nvvm_atom_max_gen_ul, "ULiULiD*ULi", "n")
 526 TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ul, "ULiULiD*ULi", "n", SM_60)
 527 TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ul, "ULiULiD*ULi", "n", SM_60)
 528 BUILTIN(__nvvm_atom_max_gen_ll, "LLiLLiD*LLi", "n")
 529 TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 530 TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 531 BUILTIN(__nvvm_atom_max_gen_ull, "ULLiULLiD*ULLi", "n")
 532 TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ull, "ULLiULLiD*ULLi", "n", SM_60)
 533 TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ull, "ULLiULLiD*ULLi", "n", SM_60)
 534
 535 BUILTIN(__nvvm_atom_min_gen_i, "iiD*i", "n")
 536 TARGET_BUILTIN(__nvvm_atom_cta_min_gen_i, "iiD*i", "n", SM_60)
 537 TARGET_BUILTIN(__nvvm_atom_sys_min_gen_i, "iiD*i", "n", SM_60)
 538 BUILTIN(__nvvm_atom_min_gen_ui, "UiUiD*Ui", "n")
 539 TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ui, "UiUiD*Ui", "n", SM_60)
 540 TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ui, "UiUiD*Ui", "n", SM_60)
 541 BUILTIN(__nvvm_atom_min_gen_l, "LiLiD*Li", "n")
 542 TARGET_BUILTIN(__nvvm_atom_cta_min_gen_l, "LiLiD*Li", "n", SM_60)
 543 TARGET_BUILTIN(__nvvm_atom_sys_min_gen_l, "LiLiD*Li", "n", SM_60)
 544 BUILTIN(__nvvm_atom_min_gen_ul, "ULiULiD*ULi", "n")
 545 TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ul, "ULiULiD*ULi", "n", SM_60)
 546 TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ul, "ULiULiD*ULi", "n", SM_60)
 547 BUILTIN(__nvvm_atom_min_gen_ll, "LLiLLiD*LLi", "n")
 548 TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 549 TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 550 BUILTIN(__nvvm_atom_min_gen_ull, "ULLiULLiD*ULLi", "n")
 551 TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ull, "ULLiULLiD*ULLi", "n", SM_60)
 552 TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ull, "ULLiULLiD*ULLi", "n", SM_60)
 553
 554 BUILTIN(__nvvm_atom_inc_gen_ui, "UiUiD*Ui", "n")
 555 TARGET_BUILTIN(__nvvm_atom_cta_inc_gen_ui, "UiUiD*Ui", "n", SM_60)
 556 TARGET_BUILTIN(__nvvm_atom_sys_inc_gen_ui, "UiUiD*Ui", "n", SM_60)
 557 BUILTIN(__nvvm_atom_dec_gen_ui, "UiUiD*Ui", "n")
 558 TARGET_BUILTIN(__nvvm_atom_cta_dec_gen_ui, "UiUiD*Ui", "n", SM_60)
 559 TARGET_BUILTIN(__nvvm_atom_sys_dec_gen_ui, "UiUiD*Ui", "n", SM_60)
 560
 561 BUILTIN(__nvvm_atom_and_gen_i, "iiD*i", "n")
 562 TARGET_BUILTIN(__nvvm_atom_cta_and_gen_i, "iiD*i", "n", SM_60)
 563 TARGET_BUILTIN(__nvvm_atom_sys_and_gen_i, "iiD*i", "n", SM_60)
 564 BUILTIN(__nvvm_atom_and_gen_l, "LiLiD*Li", "n")
 565 TARGET_BUILTIN(__nvvm_atom_cta_and_gen_l, "LiLiD*Li", "n", SM_60)
 566 TARGET_BUILTIN(__nvvm_atom_sys_and_gen_l, "LiLiD*Li", "n", SM_60)
 567 BUILTIN(__nvvm_atom_and_gen_ll, "LLiLLiD*LLi", "n")
 568 TARGET_BUILTIN(__nvvm_atom_cta_and_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 569 TARGET_BUILTIN(__nvvm_atom_sys_and_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 570
 571 BUILTIN(__nvvm_atom_or_gen_i, "iiD*i", "n")
 572 TARGET_BUILTIN(__nvvm_atom_cta_or_gen_i, "iiD*i", "n", SM_60)
 573 TARGET_BUILTIN(__nvvm_atom_sys_or_gen_i, "iiD*i", "n", SM_60)
 574 BUILTIN(__nvvm_atom_or_gen_l, "LiLiD*Li", "n")
 575 TARGET_BUILTIN(__nvvm_atom_cta_or_gen_l, "LiLiD*Li", "n", SM_60)
 576 TARGET_BUILTIN(__nvvm_atom_sys_or_gen_l, "LiLiD*Li", "n", SM_60)
 577 BUILTIN(__nvvm_atom_or_gen_ll, "LLiLLiD*LLi", "n")
 578 TARGET_BUILTIN(__nvvm_atom_cta_or_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 579 TARGET_BUILTIN(__nvvm_atom_sys_or_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 580
 581 BUILTIN(__nvvm_atom_xor_gen_i, "iiD*i", "n")
 582 TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_i, "iiD*i", "n", SM_60)
 583 TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_i, "iiD*i", "n", SM_60)
 584 BUILTIN(__nvvm_atom_xor_gen_l, "LiLiD*Li", "n")
 585 TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_l, "LiLiD*Li", "n", SM_60)
 586 TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_l, "LiLiD*Li", "n", SM_60)
 587 BUILTIN(__nvvm_atom_xor_gen_ll, "LLiLLiD*LLi", "n")
 588 TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 589 TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_ll, "LLiLLiD*LLi", "n", SM_60)
 590
 591 BUILTIN(__nvvm_atom_cas_gen_i, "iiD*ii", "n")
 592 TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_i, "iiD*ii", "n", SM_60)
 593 TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_i, "iiD*ii", "n", SM_60)
 594 BUILTIN(__nvvm_atom_cas_gen_l, "LiLiD*LiLi", "n")
 595 TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_l, "LiLiD*LiLi", "n", SM_60)
 596 TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_l, "LiLiD*LiLi", "n", SM_60)
 597 BUILTIN(__nvvm_atom_cas_gen_ll, "LLiLLiD*LLiLLi", "n")
 598 TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_ll, "LLiLLiD*LLiLLi", "n", SM_60)
 599 TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_ll, "LLiLLiD*LLiLLi", "n", SM_60)
 600
 601 // Compiler Error Warn
 602 BUILTIN(__nvvm_compiler_error, "vcC*4", "n")
 603 BUILTIN(__nvvm_compiler_warn, "vcC*4", "n")
 604
 605 // __ldg.  This is not implemented as a builtin by nvcc.
 606 BUILTIN(__nvvm_ldg_c, "ccC*", "")
 607 BUILTIN(__nvvm_ldg_s, "ssC*", "")
 608 BUILTIN(__nvvm_ldg_i, "iiC*", "")
 609 BUILTIN(__nvvm_ldg_l, "LiLiC*", "")
 610 BUILTIN(__nvvm_ldg_ll, "LLiLLiC*", "")
 611
 612 BUILTIN(__nvvm_ldg_uc, "UcUcC*", "")
 613 BUILTIN(__nvvm_ldg_us, "UsUsC*", "")
 614 BUILTIN(__nvvm_ldg_ui, "UiUiC*", "")
 615 BUILTIN(__nvvm_ldg_ul, "ULiULiC*", "")
 616 BUILTIN(__nvvm_ldg_ull, "ULLiULLiC*", "")
 617
 618 BUILTIN(__nvvm_ldg_f, "ffC*", "")
 619 BUILTIN(__nvvm_ldg_d, "ddC*", "")
 620
 621 BUILTIN(__nvvm_ldg_c2, "E2cE2cC*", "")
 622 BUILTIN(__nvvm_ldg_c4, "E4cE4cC*", "")
 623 BUILTIN(__nvvm_ldg_s2, "E2sE2sC*", "")
 624 BUILTIN(__nvvm_ldg_s4, "E4sE4sC*", "")
 625 BUILTIN(__nvvm_ldg_i2, "E2iE2iC*", "")
 626 BUILTIN(__nvvm_ldg_i4, "E4iE4iC*", "")
 627 BUILTIN(__nvvm_ldg_ll2, "E2LLiE2LLiC*", "")
 628
 629 BUILTIN(__nvvm_ldg_uc2, "E2UcE2UcC*", "")
 630 BUILTIN(__nvvm_ldg_uc4, "E4UcE4UcC*", "")
 631 BUILTIN(__nvvm_ldg_us2, "E2UsE2UsC*", "")
 632 BUILTIN(__nvvm_ldg_us4, "E4UsE4UsC*", "")
 633 BUILTIN(__nvvm_ldg_ui2, "E2UiE2UiC*", "")
 634 BUILTIN(__nvvm_ldg_ui4, "E4UiE4UiC*", "")
 635 BUILTIN(__nvvm_ldg_ull2, "E2ULLiE2ULLiC*", "")
 636
 637 BUILTIN(__nvvm_ldg_f2, "E2fE2fC*", "")
 638 BUILTIN(__nvvm_ldg_f4, "E4fE4fC*", "")
 639 BUILTIN(__nvvm_ldg_d2, "E2dE2dC*", "")
 640
 641 // Builtins to support WMMA instructions on sm_70
 642 TARGET_BUILTIN(__hmma_m16n16k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX60))
 643 TARGET_BUILTIN(__hmma_m16n16k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX60))
 644 TARGET_BUILTIN(__hmma_m16n16k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX60))
 645 TARGET_BUILTIN(__hmma_m16n16k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX60))
 646 TARGET_BUILTIN(__hmma_m16n16k16_st_c_f16, "vi*i*UiIi", "", AND(SM_70,PTX60))
 647 TARGET_BUILTIN(__hmma_m16n16k16_st_c_f32, "vf*f*UiIi", "", AND(SM_70,PTX60))
 648
 649 TARGET_BUILTIN(__hmma_m32n8k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX61))
 650 TARGET_BUILTIN(__hmma_m32n8k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX61))
 651 TARGET_BUILTIN(__hmma_m32n8k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61))
 652 TARGET_BUILTIN(__hmma_m32n8k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61))
 653 TARGET_BUILTIN(__hmma_m32n8k16_st_c_f16, "vi*i*UiIi", "", AND(SM_70,PTX61))
 654 TARGET_BUILTIN(__hmma_m32n8k16_st_c_f32, "vf*f*UiIi", "", AND(SM_70,PTX61))
 655
 656 TARGET_BUILTIN(__hmma_m8n32k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX61))
 657 TARGET_BUILTIN(__hmma_m8n32k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX61))
 658 TARGET_BUILTIN(__hmma_m8n32k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61))
 659 TARGET_BUILTIN(__hmma_m8n32k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61))
 660 TARGET_BUILTIN(__hmma_m8n32k16_st_c_f16, "vi*i*UiIi", "", AND(SM_70,PTX61))
 661 TARGET_BUILTIN(__hmma_m8n32k16_st_c_f32, "vf*f*UiIi", "", AND(SM_70,PTX61))
 662
 663 TARGET_BUILTIN(__hmma_m16n16k16_mma_f16f16, "vi*iC*iC*iC*IiIi", "", AND(SM_70,PTX60))
 664 TARGET_BUILTIN(__hmma_m16n16k16_mma_f32f16, "vf*iC*iC*iC*IiIi", "", AND(SM_70,PTX60))
 665 TARGET_BUILTIN(__hmma_m16n16k16_mma_f32f32, "vf*iC*iC*fC*IiIi", "", AND(SM_70,PTX60))
 666 TARGET_BUILTIN(__hmma_m16n16k16_mma_f16f32, "vi*iC*iC*fC*IiIi", "", AND(SM_70,PTX60))
 667
 668 TARGET_BUILTIN(__hmma_m32n8k16_mma_f16f16, "vi*iC*iC*iC*IiIi", "", AND(SM_70,PTX61))
 669 TARGET_BUILTIN(__hmma_m32n8k16_mma_f32f16, "vf*iC*iC*iC*IiIi", "", AND(SM_70,PTX61))
 670 TARGET_BUILTIN(__hmma_m32n8k16_mma_f32f32, "vf*iC*iC*fC*IiIi", "", AND(SM_70,PTX61))
 671 TARGET_BUILTIN(__hmma_m32n8k16_mma_f16f32, "vi*iC*iC*fC*IiIi", "", AND(SM_70,PTX61))
 672
 673 TARGET_BUILTIN(__hmma_m8n32k16_mma_f16f16, "vi*iC*iC*iC*IiIi", "", AND(SM_70,PTX61))
 674 TARGET_BUILTIN(__hmma_m8n32k16_mma_f32f16, "vf*iC*iC*iC*IiIi", "", AND(SM_70,PTX61))
 675 TARGET_BUILTIN(__hmma_m8n32k16_mma_f32f32, "vf*iC*iC*fC*IiIi", "", AND(SM_70,PTX61))
 676 TARGET_BUILTIN(__hmma_m8n32k16_mma_f16f32, "vi*iC*iC*fC*IiIi", "", AND(SM_70,PTX61))
 677
 678 // Builtins to support integer and sub-integer WMMA instructions on sm_72/sm_75
 679 TARGET_BUILTIN(__bmma_m8n8k128_ld_a_b1, "vi*iC*UiIi", "", AND(SM_75,PTX63))
 680 TARGET_BUILTIN(__bmma_m8n8k128_ld_b_b1, "vi*iC*UiIi", "", AND(SM_75,PTX63))
 681 TARGET_BUILTIN(__bmma_m8n8k128_ld_c, "vi*iC*UiIi", "", AND(SM_75,PTX63))
 682 TARGET_BUILTIN(__bmma_m8n8k128_mma_xor_popc_b1, "vi*iC*iC*iC*Ii", "", AND(SM_75,PTX63))
 683 TARGET_BUILTIN(__bmma_m8n8k128_st_c_i32, "vi*iC*UiIi", "", AND(SM_75,PTX63))
 684 TARGET_BUILTIN(__imma_m16n16k16_ld_a_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 685 TARGET_BUILTIN(__imma_m16n16k16_ld_a_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 686 TARGET_BUILTIN(__imma_m16n16k16_ld_b_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 687 TARGET_BUILTIN(__imma_m16n16k16_ld_b_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 688 TARGET_BUILTIN(__imma_m16n16k16_ld_c, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 689 TARGET_BUILTIN(__imma_m16n16k16_mma_s8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
 690 TARGET_BUILTIN(__imma_m16n16k16_mma_u8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
 691 TARGET_BUILTIN(__imma_m16n16k16_st_c_i32, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 692 TARGET_BUILTIN(__imma_m32n8k16_ld_a_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 693 TARGET_BUILTIN(__imma_m32n8k16_ld_a_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 694 TARGET_BUILTIN(__imma_m32n8k16_ld_b_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 695 TARGET_BUILTIN(__imma_m32n8k16_ld_b_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 696 TARGET_BUILTIN(__imma_m32n8k16_ld_c, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 697 TARGET_BUILTIN(__imma_m32n8k16_mma_s8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
 698 TARGET_BUILTIN(__imma_m32n8k16_mma_u8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
 699 TARGET_BUILTIN(__imma_m32n8k16_st_c_i32, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 700 TARGET_BUILTIN(__imma_m8n32k16_ld_a_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 701 TARGET_BUILTIN(__imma_m8n32k16_ld_a_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 702 TARGET_BUILTIN(__imma_m8n32k16_ld_b_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 703 TARGET_BUILTIN(__imma_m8n32k16_ld_b_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 704 TARGET_BUILTIN(__imma_m8n32k16_ld_c, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 705 TARGET_BUILTIN(__imma_m8n32k16_mma_s8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
 706 TARGET_BUILTIN(__imma_m8n32k16_mma_u8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63))
 707 TARGET_BUILTIN(__imma_m8n32k16_st_c_i32, "vi*iC*UiIi", "", AND(SM_72,PTX63))
 708 TARGET_BUILTIN(__imma_m8n8k32_ld_a_s4, "vi*iC*UiIi", "", AND(SM_75,PTX63))
 709 TARGET_BUILTIN(__imma_m8n8k32_ld_a_u4, "vi*iC*UiIi", "", AND(SM_75,PTX63))
 710 TARGET_BUILTIN(__imma_m8n8k32_ld_b_s4, "vi*iC*UiIi", "", AND(SM_75,PTX63))
 711 TARGET_BUILTIN(__imma_m8n8k32_ld_b_u4, "vi*iC*UiIi", "", AND(SM_75,PTX63))
 712 TARGET_BUILTIN(__imma_m8n8k32_ld_c, "vi*iC*UiIi", "", AND(SM_75,PTX63))
 713 TARGET_BUILTIN(__imma_m8n8k32_mma_s4, "vi*iC*iC*iC*IiIi", "", AND(SM_75,PTX63))
 714 TARGET_BUILTIN(__imma_m8n8k32_mma_u4, "vi*iC*iC*iC*IiIi", "", AND(SM_75,PTX63))
 715 TARGET_BUILTIN(__imma_m8n8k32_st_c_i32, "vi*iC*UiIi", "", AND(SM_75,PTX63))
 716
 717 #undef BUILTIN
 718 #undef TARGET_BUILTIN
 719 #pragma pop_macro("AND")
 720 #pragma pop_macro("SM_60")
 721 #pragma pop_macro("SM_70")
 722 #pragma pop_macro("SM_72")
 723 #pragma pop_macro("SM_75")
 724 #pragma pop_macro("PTX60")
 725 #pragma pop_macro("PTX61")
 726 #pragma pop_macro("PTX63")
 727 #pragma pop_macro("PTX64")