contrib/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td

   1 //===- NVPTXIntrinsics.td - PTX Intrinsics Instructions -------*- tblgen -*-==//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9
  10 def immFloat0 : PatLeaf<(fpimm), [{
  11     float f = (float)N->getValueAPF().convertToFloat();
  12     return (f==0.0f);
  13 }]>;
  14
  15 def immFloat1 : PatLeaf<(fpimm), [{
  16     float f = (float)N->getValueAPF().convertToFloat();
  17     return (f==1.0f);
  18 }]>;
  19
  20 def immDouble0 : PatLeaf<(fpimm), [{
  21     double d = (double)N->getValueAPF().convertToDouble();
  22     return (d==0.0);
  23 }]>;
  24
  25 def immDouble1 : PatLeaf<(fpimm), [{
  26     double d = (double)N->getValueAPF().convertToDouble();
  27     return (d==1.0);
  28 }]>;
  29
  30
  31
  32 //-----------------------------------
  33 // Synchronization and shuffle functions
  34 //-----------------------------------
  35 let isConvergent = 1 in {
  36 def INT_BARRIER0 : NVPTXInst<(outs), (ins),
  37                   "bar.sync \t0;",
  38       [(int_nvvm_barrier0)]>;
  39 def INT_BARRIERN : NVPTXInst<(outs), (ins Int32Regs:$src1),
  40                   "bar.sync \t$src1;",
  41       [(int_nvvm_barrier_n Int32Regs:$src1)]>;
  42 def INT_BARRIER : NVPTXInst<(outs), (ins Int32Regs:$src1, Int32Regs:$src2),
  43                   "bar.sync \t$src1, $src2;",
  44       [(int_nvvm_barrier Int32Regs:$src1, Int32Regs:$src2)]>;
  45 def INT_BARRIER0_POPC : NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$pred),
  46   !strconcat("{{ \n\t",
  47              ".reg .pred \t%p1; \n\t",
  48              "setp.ne.u32 \t%p1, $pred, 0; \n\t",
  49              "bar.red.popc.u32 \t$dst, 0, %p1; \n\t",
  50              "}}"),
  51       [(set Int32Regs:$dst, (int_nvvm_barrier0_popc Int32Regs:$pred))]>;
  52 def INT_BARRIER0_AND : NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$pred),
  53   !strconcat("{{ \n\t",
  54              ".reg .pred \t%p1; \n\t",
  55              ".reg .pred \t%p2; \n\t",
  56              "setp.ne.u32 \t%p1, $pred, 0; \n\t",
  57              "bar.red.and.pred \t%p2, 0, %p1; \n\t",
  58              "selp.u32 \t$dst, 1, 0, %p2; \n\t",
  59              "}}"),
  60       [(set Int32Regs:$dst, (int_nvvm_barrier0_and Int32Regs:$pred))]>;
  61 def INT_BARRIER0_OR : NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$pred),
  62   !strconcat("{{ \n\t",
  63              ".reg .pred \t%p1; \n\t",
  64              ".reg .pred \t%p2; \n\t",
  65              "setp.ne.u32 \t%p1, $pred, 0; \n\t",
  66              "bar.red.or.pred \t%p2, 0, %p1; \n\t",
  67              "selp.u32 \t$dst, 1, 0, %p2; \n\t",
  68              "}}"),
  69       [(set Int32Regs:$dst, (int_nvvm_barrier0_or Int32Regs:$pred))]>;
  70
  71 def INT_BAR_SYNC : NVPTXInst<(outs), (ins i32imm:$i), "bar.sync \t$i;",
  72                              [(int_nvvm_bar_sync imm:$i)]>;
  73
  74 def INT_BAR_WARP_SYNC_I : NVPTXInst<(outs), (ins i32imm:$i), "bar.warp.sync \t$i;",
  75                              [(int_nvvm_bar_warp_sync imm:$i)]>,
  76         Requires<[hasPTX60, hasSM30]>;
  77 def INT_BAR_WARP_SYNC_R : NVPTXInst<(outs), (ins Int32Regs:$i), "bar.warp.sync \t$i;",
  78                              [(int_nvvm_bar_warp_sync Int32Regs:$i)]>,
  79         Requires<[hasPTX60, hasSM30]>;
  80
  81 def INT_BARRIER_SYNC_I : NVPTXInst<(outs), (ins i32imm:$i), "barrier.sync \t$i;",
  82                                    [(int_nvvm_barrier_sync imm:$i)]>,
  83         Requires<[hasPTX60, hasSM30]>;
  84 def INT_BARRIER_SYNC_R : NVPTXInst<(outs), (ins Int32Regs:$i), "barrier.sync \t$i;",
  85                                    [(int_nvvm_barrier_sync Int32Regs:$i)]>,
  86         Requires<[hasPTX60, hasSM30]>;
  87
  88 def INT_BARRIER_SYNC_CNT_RR : NVPTXInst<(outs), (ins Int32Regs:$id, Int32Regs:$cnt),
  89                  "barrier.sync \t$id, $cnt;",
  90                  [(int_nvvm_barrier_sync_cnt Int32Regs:$id, Int32Regs:$cnt)]>,
  91         Requires<[hasPTX60, hasSM30]>;
  92 def INT_BARRIER_SYNC_CNT_RI : NVPTXInst<(outs), (ins Int32Regs:$id, i32imm:$cnt),
  93                  "barrier.sync \t$id, $cnt;",
  94                  [(int_nvvm_barrier_sync_cnt Int32Regs:$id, imm:$cnt)]>,
  95         Requires<[hasPTX60, hasSM30]>;
  96 def INT_BARRIER_SYNC_CNT_IR : NVPTXInst<(outs), (ins i32imm:$id, Int32Regs:$cnt),
  97                  "barrier.sync \t$id, $cnt;",
  98                  [(int_nvvm_barrier_sync_cnt imm:$id, Int32Regs:$cnt)]>,
  99         Requires<[hasPTX60, hasSM30]>;
 100 def INT_BARRIER_SYNC_CNT_II : NVPTXInst<(outs), (ins i32imm:$id, i32imm:$cnt),
 101                  "barrier.sync \t$id, $cnt;",
 102                  [(int_nvvm_barrier_sync_cnt imm:$id, imm:$cnt)]>,
 103         Requires<[hasPTX60, hasSM30]>;
 104
 105
 106 // shfl.{up,down,bfly,idx}.b32
 107 multiclass SHFL<NVPTXRegClass regclass, string mode, Intrinsic IntOp> {
 108   // The last two parameters to shfl can be regs or imms.  ptxas is smart
 109   // enough to inline constant registers, so strictly speaking we don't need to
 110   // handle immediates here.  But it's easy enough, and it makes our ptx more
 111   // readable.
 112   def reg : NVPTXInst<
 113       (outs regclass:$dst),
 114       (ins regclass:$src, Int32Regs:$offset, Int32Regs:$mask),
 115       !strconcat("shfl.", mode, ".b32 $dst, $src, $offset, $mask;"),
 116       [(set regclass:$dst, (IntOp regclass:$src, Int32Regs:$offset, Int32Regs:$mask))]>;
 117
 118   def imm1 : NVPTXInst<
 119       (outs regclass:$dst),
 120       (ins regclass:$src, i32imm:$offset, Int32Regs:$mask),
 121       !strconcat("shfl.", mode, ".b32 $dst, $src, $offset, $mask;"),
 122       [(set regclass:$dst, (IntOp regclass:$src, imm:$offset, Int32Regs:$mask))]>;
 123
 124   def imm2 : NVPTXInst<
 125       (outs regclass:$dst),
 126       (ins regclass:$src, Int32Regs:$offset, i32imm:$mask),
 127       !strconcat("shfl.", mode, ".b32 $dst, $src, $offset, $mask;"),
 128       [(set regclass:$dst, (IntOp regclass:$src, Int32Regs:$offset, imm:$mask))]>;
 129
 130   def imm3 : NVPTXInst<
 131       (outs regclass:$dst),
 132       (ins regclass:$src, i32imm:$offset, i32imm:$mask),
 133       !strconcat("shfl.", mode, ".b32 $dst, $src, $offset, $mask;"),
 134       [(set regclass:$dst, (IntOp regclass:$src, imm:$offset, imm:$mask))]>;
 135 }
 136
 137 defm INT_SHFL_DOWN_I32 : SHFL<Int32Regs, "down", int_nvvm_shfl_down_i32>;
 138 defm INT_SHFL_DOWN_F32 : SHFL<Float32Regs, "down", int_nvvm_shfl_down_f32>;
 139 defm INT_SHFL_UP_I32 : SHFL<Int32Regs, "up", int_nvvm_shfl_up_i32>;
 140 defm INT_SHFL_UP_F32 : SHFL<Float32Regs, "up", int_nvvm_shfl_up_f32>;
 141 defm INT_SHFL_BFLY_I32 : SHFL<Int32Regs, "bfly", int_nvvm_shfl_bfly_i32>;
 142 defm INT_SHFL_BFLY_F32 : SHFL<Float32Regs, "bfly", int_nvvm_shfl_bfly_f32>;
 143 defm INT_SHFL_IDX_I32 : SHFL<Int32Regs, "idx", int_nvvm_shfl_idx_i32>;
 144 defm INT_SHFL_IDX_F32 : SHFL<Float32Regs, "idx", int_nvvm_shfl_idx_f32>;
 145
 146 multiclass SHFL_SYNC<NVPTXRegClass regclass, string mode, Intrinsic IntOp> {
 147   // Threadmask and the last two parameters to shfl.sync can be regs or imms.
 148   // ptxas is smart enough to inline constant registers, so strictly speaking we
 149   // don't need to handle immediates here.  But it's easy enough, and it makes
 150   // our ptx more readable.
 151   def rrr : NVPTXInst<
 152       (outs regclass:$dst),
 153       (ins Int32Regs:$threadmask, regclass:$src, Int32Regs:$offset, Int32Regs:$mask),
 154       !strconcat("shfl.sync.", mode, ".b32 $dst, $src, $offset, $mask, $threadmask;"),
 155       [(set regclass:$dst, (IntOp Int32Regs:$threadmask, regclass:$src,
 156                             Int32Regs:$offset, Int32Regs:$mask))]>;
 157
 158   def rri : NVPTXInst<
 159       (outs regclass:$dst),
 160       (ins Int32Regs:$threadmask, regclass:$src, Int32Regs:$offset, i32imm:$mask),
 161       !strconcat("shfl.sync.", mode, ".b32 $dst, $src, $offset, $mask, $threadmask;"),
 162       [(set regclass:$dst, (IntOp Int32Regs:$threadmask, regclass:$src,
 163                             Int32Regs:$offset, imm:$mask))]>;
 164
 165   def rir : NVPTXInst<
 166       (outs regclass:$dst),
 167       (ins Int32Regs:$threadmask, regclass:$src, i32imm:$offset, Int32Regs:$mask),
 168       !strconcat("shfl.sync.", mode, ".b32 $dst, $src, $offset, $mask, $threadmask;"),
 169       [(set regclass:$dst, (IntOp Int32Regs:$threadmask, regclass:$src,
 170                             imm:$offset, Int32Regs:$mask))]>;
 171
 172   def rii : NVPTXInst<
 173       (outs regclass:$dst),
 174       (ins Int32Regs:$threadmask, regclass:$src, i32imm:$offset, i32imm:$mask),
 175       !strconcat("shfl.sync.", mode, ".b32 $dst, $src, $offset, $mask, $threadmask;"),
 176       [(set regclass:$dst, (IntOp Int32Regs:$threadmask, regclass:$src,
 177                             imm:$offset, imm:$mask))]>;
 178
 179   def irr : NVPTXInst<
 180       (outs regclass:$dst),
 181       (ins i32imm:$threadmask, regclass:$src, Int32Regs:$offset, Int32Regs:$mask),
 182       !strconcat("shfl.sync.", mode, ".b32 $dst, $src, $offset, $mask, $threadmask;"),
 183       [(set regclass:$dst, (IntOp imm:$threadmask, regclass:$src,
 184                             Int32Regs:$offset, Int32Regs:$mask))]>;
 185
 186   def iri : NVPTXInst<
 187       (outs regclass:$dst),
 188       (ins i32imm:$threadmask, regclass:$src, Int32Regs:$offset, i32imm:$mask),
 189       !strconcat("shfl.sync.", mode, ".b32 $dst, $src, $offset, $mask, $threadmask;"),
 190       [(set regclass:$dst, (IntOp imm:$threadmask, regclass:$src,
 191                             Int32Regs:$offset, imm:$mask))]>;
 192
 193   def iir : NVPTXInst<
 194       (outs regclass:$dst),
 195       (ins i32imm:$threadmask, regclass:$src, i32imm:$offset, Int32Regs:$mask),
 196       !strconcat("shfl.sync.", mode, ".b32 $dst, $src, $offset, $mask, $threadmask;"),
 197       [(set regclass:$dst, (IntOp imm:$threadmask, regclass:$src,
 198                             imm:$offset, Int32Regs:$mask))]>;
 199
 200   def iii : NVPTXInst<
 201       (outs regclass:$dst),
 202       (ins i32imm:$threadmask, regclass:$src, i32imm:$offset, i32imm:$mask),
 203       !strconcat("shfl.sync.", mode, ".b32 $dst, $src, $offset, $mask, $threadmask;"),
 204       [(set regclass:$dst, (IntOp imm:$threadmask, regclass:$src,
 205                             imm:$offset, imm:$mask))]>;
 206 }
 207
 208 // On sm_70 these don't have to be convergent, so we may eventually want to
 209 // implement non-convergent variant of this intrinsic.
 210 defm INT_SHFL_SYNC_DOWN_I32 : SHFL_SYNC<Int32Regs, "down", int_nvvm_shfl_sync_down_i32>;
 211 defm INT_SHFL_SYNC_DOWN_F32 : SHFL_SYNC<Float32Regs, "down", int_nvvm_shfl_sync_down_f32>;
 212 defm INT_SHFL_SYNC_UP_I32 : SHFL_SYNC<Int32Regs, "up", int_nvvm_shfl_sync_up_i32>;
 213 defm INT_SHFL_SYNC_UP_F32 : SHFL_SYNC<Float32Regs, "up", int_nvvm_shfl_sync_up_f32>;
 214 defm INT_SHFL_SYNC_BFLY_I32 : SHFL_SYNC<Int32Regs, "bfly", int_nvvm_shfl_sync_bfly_i32>;
 215 defm INT_SHFL_SYNC_BFLY_F32 : SHFL_SYNC<Float32Regs, "bfly", int_nvvm_shfl_sync_bfly_f32>;
 216 defm INT_SHFL_SYNC_IDX_I32 : SHFL_SYNC<Int32Regs, "idx", int_nvvm_shfl_sync_idx_i32>;
 217 defm INT_SHFL_SYNC_IDX_F32 : SHFL_SYNC<Float32Regs, "idx", int_nvvm_shfl_sync_idx_f32>;
 218
 219
 220 // vote.{all,any,uni,ballot}
 221 multiclass VOTE<NVPTXRegClass regclass, string mode, Intrinsic IntOp> {
 222   def : NVPTXInst<(outs regclass:$dest), (ins Int1Regs:$pred),
 223               "vote." # mode # " \t$dest, $pred;",
 224               [(set regclass:$dest, (IntOp Int1Regs:$pred))]>,
 225         Requires<[hasPTX60, hasSM30]>;
 226 }
 227
 228 defm VOTE_ALL : VOTE<Int1Regs, "all.pred", int_nvvm_vote_all>;
 229 defm VOTE_ANY : VOTE<Int1Regs, "any.pred", int_nvvm_vote_any>;
 230 defm VOTE_UNI : VOTE<Int1Regs, "uni.pred", int_nvvm_vote_uni>;
 231 defm VOTE_BALLOT : VOTE<Int32Regs, "ballot.b32", int_nvvm_vote_ballot>;
 232
 233 // vote.sync.{all,any,uni,ballot}
 234 multiclass VOTE_SYNC<NVPTXRegClass regclass, string mode, Intrinsic IntOp> {
 235   def i : NVPTXInst<(outs regclass:$dest), (ins i32imm:$mask, Int1Regs:$pred),
 236               "vote.sync." # mode # " \t$dest, $pred, $mask;",
 237               [(set regclass:$dest, (IntOp imm:$mask, Int1Regs:$pred))]>,
 238           Requires<[hasPTX60, hasSM30]>;
 239   def r : NVPTXInst<(outs regclass:$dest), (ins Int32Regs:$mask, Int1Regs:$pred),
 240               "vote.sync." # mode #" \t$dest, $pred, $mask;",
 241               [(set regclass:$dest, (IntOp Int32Regs:$mask, Int1Regs:$pred))]>,
 242           Requires<[hasPTX60, hasSM30]>;
 243 }
 244
 245 defm VOTE_SYNC_ALL : VOTE_SYNC<Int1Regs, "all.pred", int_nvvm_vote_all_sync>;
 246 defm VOTE_SYNC_ANY : VOTE_SYNC<Int1Regs, "any.pred", int_nvvm_vote_any_sync>;
 247 defm VOTE_SYNC_UNI : VOTE_SYNC<Int1Regs, "uni.pred", int_nvvm_vote_uni_sync>;
 248 defm VOTE_SYNC_BALLOT : VOTE_SYNC<Int32Regs, "ballot.b32", int_nvvm_vote_ballot_sync>;
 249
 250 multiclass MATCH_ANY_SYNC<NVPTXRegClass regclass, string ptxtype, Intrinsic IntOp,
 251                           Operand ImmOp> {
 252   def ii : NVPTXInst<(outs regclass:$dest), (ins i32imm:$mask, ImmOp:$value),
 253               "match.any.sync." # ptxtype # " \t$dest, $value, $mask;",
 254               [(set regclass:$dest, (IntOp imm:$mask, imm:$value))]>,
 255            Requires<[hasPTX60, hasSM70]>;
 256   def ir : NVPTXInst<(outs regclass:$dest), (ins Int32Regs:$mask, ImmOp:$value),
 257               "match.any.sync." # ptxtype # " \t$dest, $value, $mask;",
 258               [(set regclass:$dest, (IntOp Int32Regs:$mask, imm:$value))]>,
 259            Requires<[hasPTX60, hasSM70]>;
 260   def ri : NVPTXInst<(outs regclass:$dest), (ins i32imm:$mask, regclass:$value),
 261               "match.any.sync." # ptxtype # " \t$dest, $value, $mask;",
 262               [(set regclass:$dest, (IntOp imm:$mask, regclass:$value))]>,
 263            Requires<[hasPTX60, hasSM70]>;
 264   def rr : NVPTXInst<(outs regclass:$dest), (ins Int32Regs:$mask, regclass:$value),
 265               "match.any.sync." # ptxtype # " \t$dest, $value, $mask;",
 266               [(set regclass:$dest, (IntOp Int32Regs:$mask, regclass:$value))]>,
 267            Requires<[hasPTX60, hasSM70]>;
 268 }
 269
 270 defm MATCH_ANY_SYNC_32 : MATCH_ANY_SYNC<Int32Regs, "b32", int_nvvm_match_any_sync_i32,
 271                                         i32imm>;
 272 defm MATCH_ANY_SYNC_64 : MATCH_ANY_SYNC<Int64Regs, "b64", int_nvvm_match_any_sync_i64,
 273                                         i64imm>;
 274
 275 multiclass MATCH_ALLP_SYNC<NVPTXRegClass regclass, string ptxtype, Intrinsic IntOp,
 276                           Operand ImmOp> {
 277   def ii : NVPTXInst<(outs regclass:$dest, Int1Regs:$pred),
 278                      (ins i32imm:$mask, ImmOp:$value),
 279               "match.all.sync." # ptxtype # " \t$dest|$pred, $value, $mask;",
 280               [(set regclass:$dest, Int1Regs:$pred, (IntOp imm:$mask, imm:$value))]>,
 281            Requires<[hasPTX60, hasSM70]>;
 282   def ir : NVPTXInst<(outs regclass:$dest, Int1Regs:$pred),
 283                      (ins Int32Regs:$mask, ImmOp:$value),
 284               "match.all.sync." # ptxtype # " \t$dest|$pred, $value, $mask;",
 285               [(set regclass:$dest, Int1Regs:$pred, (IntOp Int32Regs:$mask, imm:$value))]>,
 286            Requires<[hasPTX60, hasSM70]>;
 287   def ri : NVPTXInst<(outs regclass:$dest, Int1Regs:$pred),
 288                      (ins i32imm:$mask, regclass:$value),
 289               "match.all.sync." # ptxtype # " \t$dest|$pred, $value, $mask;",
 290               [(set regclass:$dest, Int1Regs:$pred, (IntOp imm:$mask, regclass:$value))]>,
 291            Requires<[hasPTX60, hasSM70]>;
 292   def rr : NVPTXInst<(outs regclass:$dest, Int1Regs:$pred),
 293                      (ins Int32Regs:$mask, regclass:$value),
 294               "match.all.sync." # ptxtype # " \t$dest|$pred, $value, $mask;",
 295               [(set regclass:$dest, Int1Regs:$pred, (IntOp Int32Regs:$mask, regclass:$value))]>,
 296            Requires<[hasPTX60, hasSM70]>;
 297 }
 298 defm MATCH_ALLP_SYNC_32 : MATCH_ALLP_SYNC<Int32Regs, "b32", int_nvvm_match_all_sync_i32p,
 299                                          i32imm>;
 300 defm MATCH_ALLP_SYNC_64 : MATCH_ALLP_SYNC<Int64Regs, "b64", int_nvvm_match_all_sync_i64p,
 301                                          i64imm>;
 302
 303 } // isConvergent = 1
 304
 305 //-----------------------------------
 306 // Explicit Memory Fence Functions
 307 //-----------------------------------
 308 class MEMBAR<string StrOp, Intrinsic IntOP> :
 309               NVPTXInst<(outs), (ins),
 310             StrOp, [(IntOP)]>;
 311
 312 def INT_MEMBAR_CTA : MEMBAR<"membar.cta;", int_nvvm_membar_cta>;
 313 def INT_MEMBAR_GL  : MEMBAR<"membar.gl;",  int_nvvm_membar_gl>;
 314 def INT_MEMBAR_SYS : MEMBAR<"membar.sys;", int_nvvm_membar_sys>;
 315
 316
 317 //-----------------------------------
 318 // Math Functions
 319 //-----------------------------------
 320
 321 // Map min(1.0, max(0.0, x)) to sat(x)
 322 // Note that max(0.0, min(x, 1.0)) cannot be mapped to sat(x) because when x is
 323 // NaN
 324 // max(0.0, min(x, 1.0)) is 1.0 while sat(x) is 0.
 325 // Same story for fmax, fmin.
 326
 327 def : Pat<(int_nvvm_fmin_f immFloat1,
 328             (int_nvvm_fmax_f immFloat0, Float32Regs:$a)),
 329           (CVT_f32_f32 Float32Regs:$a, CvtSAT)>;
 330 def : Pat<(int_nvvm_fmin_f immFloat1,
 331             (int_nvvm_fmax_f Float32Regs:$a, immFloat0)),
 332           (CVT_f32_f32 Float32Regs:$a, CvtSAT)>;
 333 def : Pat<(int_nvvm_fmin_f
 334             (int_nvvm_fmax_f immFloat0, Float32Regs:$a), immFloat1),
 335           (CVT_f32_f32 Float32Regs:$a, CvtSAT)>;
 336 def : Pat<(int_nvvm_fmin_f
 337             (int_nvvm_fmax_f Float32Regs:$a, immFloat0), immFloat1),
 338           (CVT_f32_f32 Float32Regs:$a, CvtSAT)>;
 339
 340 def : Pat<(int_nvvm_fmin_d immDouble1,
 341             (int_nvvm_fmax_d immDouble0, Float64Regs:$a)),
 342           (CVT_f64_f64 Float64Regs:$a, CvtSAT)>;
 343 def : Pat<(int_nvvm_fmin_d immDouble1,
 344             (int_nvvm_fmax_d Float64Regs:$a, immDouble0)),
 345           (CVT_f64_f64 Float64Regs:$a, CvtSAT)>;
 346 def : Pat<(int_nvvm_fmin_d
 347             (int_nvvm_fmax_d immDouble0, Float64Regs:$a), immDouble1),
 348           (CVT_f64_f64 Float64Regs:$a, CvtSAT)>;
 349 def : Pat<(int_nvvm_fmin_d
 350             (int_nvvm_fmax_d Float64Regs:$a, immDouble0), immDouble1),
 351           (CVT_f64_f64 Float64Regs:$a, CvtSAT)>;
 352
 353
 354 // We need a full string for OpcStr here because we need to deal with case like
 355 // INT_PTX_RECIP.
 356 class F_MATH_1<string OpcStr, NVPTXRegClass target_regclass,
 357   NVPTXRegClass src_regclass, Intrinsic IntOP>
 358             : NVPTXInst<(outs target_regclass:$dst), (ins src_regclass:$src0),
 359             OpcStr,
 360         [(set target_regclass:$dst, (IntOP src_regclass:$src0))]>;
 361
 362 // We need a full string for OpcStr here because we need to deal with the case
 363 // like INT_PTX_NATIVE_POWR_F.
 364 class F_MATH_2<string OpcStr, NVPTXRegClass t_regclass,
 365   NVPTXRegClass s0_regclass, NVPTXRegClass s1_regclass, Intrinsic IntOP>
 366             : NVPTXInst<(outs t_regclass:$dst),
 367               (ins s0_regclass:$src0, s1_regclass:$src1),
 368             OpcStr,
 369         [(set t_regclass:$dst, (IntOP s0_regclass:$src0, s1_regclass:$src1))]>;
 370
 371 class F_MATH_3<string OpcStr, NVPTXRegClass t_regclass,
 372   NVPTXRegClass s0_regclass, NVPTXRegClass s1_regclass,
 373   NVPTXRegClass s2_regclass, Intrinsic IntOP>
 374             : NVPTXInst<(outs t_regclass:$dst),
 375               (ins s0_regclass:$src0, s1_regclass:$src1, s2_regclass:$src2),
 376             OpcStr,
 377         [(set t_regclass:$dst,
 378           (IntOP s0_regclass:$src0, s1_regclass:$src1, s2_regclass:$src2))]>;
 379
 380 //
 381 // MISC
 382 //
 383
 384 def INT_NVVM_PRMT : F_MATH_3<"prmt.b32 \t$dst, $src0, $src1, $src2;", Int32Regs,
 385   Int32Regs, Int32Regs, Int32Regs, int_nvvm_prmt>;
 386
 387 //
 388 // Min Max
 389 //
 390
 391 def INT_NVVM_FMIN_F : F_MATH_2<"min.f32 \t$dst, $src0, $src1;", Float32Regs,
 392   Float32Regs, Float32Regs, int_nvvm_fmin_f>;
 393 def INT_NVVM_FMIN_FTZ_F : F_MATH_2<"min.ftz.f32 \t$dst, $src0, $src1;",
 394   Float32Regs, Float32Regs, Float32Regs, int_nvvm_fmin_ftz_f>;
 395
 396 def INT_NVVM_FMAX_F : F_MATH_2<"max.f32 \t$dst, $src0, $src1;", Float32Regs,
 397   Float32Regs, Float32Regs, int_nvvm_fmax_f>;
 398 def INT_NVVM_FMAX_FTZ_F : F_MATH_2<"max.ftz.f32 \t$dst, $src0, $src1;",
 399   Float32Regs, Float32Regs, Float32Regs, int_nvvm_fmax_ftz_f>;
 400
 401 def INT_NVVM_FMIN_D : F_MATH_2<"min.f64 \t$dst, $src0, $src1;", Float64Regs,
 402   Float64Regs, Float64Regs, int_nvvm_fmin_d>;
 403 def INT_NVVM_FMAX_D : F_MATH_2<"max.f64 \t$dst, $src0, $src1;", Float64Regs,
 404   Float64Regs, Float64Regs, int_nvvm_fmax_d>;
 405
 406
 407 //
 408 // Multiplication
 409 //
 410
 411 def INT_NVVM_MULHI_I : F_MATH_2<"mul.hi.s32 \t$dst, $src0, $src1;", Int32Regs,
 412   Int32Regs, Int32Regs, int_nvvm_mulhi_i>;
 413 def INT_NVVM_MULHI_UI : F_MATH_2<"mul.hi.u32 \t$dst, $src0, $src1;", Int32Regs,
 414   Int32Regs, Int32Regs, int_nvvm_mulhi_ui>;
 415
 416 def INT_NVVM_MULHI_LL : F_MATH_2<"mul.hi.s64 \t$dst, $src0, $src1;", Int64Regs,
 417   Int64Regs, Int64Regs, int_nvvm_mulhi_ll>;
 418 def INT_NVVM_MULHI_ULL : F_MATH_2<"mul.hi.u64 \t$dst, $src0, $src1;", Int64Regs,
 419   Int64Regs, Int64Regs, int_nvvm_mulhi_ull>;
 420
 421 def INT_NVVM_MUL_RN_FTZ_F : F_MATH_2<"mul.rn.ftz.f32 \t$dst, $src0, $src1;",
 422   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rn_ftz_f>;
 423 def INT_NVVM_MUL_RN_F : F_MATH_2<"mul.rn.f32 \t$dst, $src0, $src1;",
 424   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rn_f>;
 425 def INT_NVVM_MUL_RZ_FTZ_F : F_MATH_2<"mul.rz.ftz.f32 \t$dst, $src0, $src1;",
 426   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rz_ftz_f>;
 427 def INT_NVVM_MUL_RZ_F : F_MATH_2<"mul.rz.f32 \t$dst, $src0, $src1;",
 428   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rz_f>;
 429 def INT_NVVM_MUL_RM_FTZ_F : F_MATH_2<"mul.rm.ftz.f32 \t$dst, $src0, $src1;",
 430   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rm_ftz_f>;
 431 def INT_NVVM_MUL_RM_F : F_MATH_2<"mul.rm.f32 \t$dst, $src0, $src1;",
 432   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rm_f>;
 433 def INT_NVVM_MUL_RP_FTZ_F : F_MATH_2<"mul.rp.ftz.f32 \t$dst, $src0, $src1;",
 434   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rp_ftz_f>;
 435 def INT_NVVM_MUL_RP_F : F_MATH_2<"mul.rp.f32 \t$dst, $src0, $src1;",
 436   Float32Regs, Float32Regs, Float32Regs, int_nvvm_mul_rp_f>;
 437
 438 def INT_NVVM_MUL_RN_D : F_MATH_2<"mul.rn.f64 \t$dst, $src0, $src1;",
 439   Float64Regs, Float64Regs, Float64Regs, int_nvvm_mul_rn_d>;
 440 def INT_NVVM_MUL_RZ_D : F_MATH_2<"mul.rz.f64 \t$dst, $src0, $src1;",
 441   Float64Regs, Float64Regs, Float64Regs, int_nvvm_mul_rz_d>;
 442 def INT_NVVM_MUL_RM_D : F_MATH_2<"mul.rm.f64 \t$dst, $src0, $src1;",
 443   Float64Regs, Float64Regs, Float64Regs, int_nvvm_mul_rm_d>;
 444 def INT_NVVM_MUL_RP_D : F_MATH_2<"mul.rp.f64 \t$dst, $src0, $src1;",
 445   Float64Regs, Float64Regs, Float64Regs, int_nvvm_mul_rp_d>;
 446
 447 def INT_NVVM_MUL24_I : F_MATH_2<"mul24.lo.s32 \t$dst, $src0, $src1;",
 448   Int32Regs, Int32Regs, Int32Regs, int_nvvm_mul24_i>;
 449 def INT_NVVM_MUL24_UI : F_MATH_2<"mul24.lo.u32 \t$dst, $src0, $src1;",
 450   Int32Regs, Int32Regs, Int32Regs, int_nvvm_mul24_ui>;
 451
 452 //
 453 // Div
 454 //
 455
 456 def INT_NVVM_DIV_APPROX_FTZ_F
 457   : F_MATH_2<"div.approx.ftz.f32 \t$dst, $src0, $src1;", Float32Regs,
 458     Float32Regs, Float32Regs, int_nvvm_div_approx_ftz_f>;
 459 def INT_NVVM_DIV_APPROX_F : F_MATH_2<"div.approx.f32 \t$dst, $src0, $src1;",
 460   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_approx_f>;
 461
 462 def INT_NVVM_DIV_RN_FTZ_F : F_MATH_2<"div.rn.ftz.f32 \t$dst, $src0, $src1;",
 463   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rn_ftz_f>;
 464 def INT_NVVM_DIV_RN_F     : F_MATH_2<"div.rn.f32 \t$dst, $src0, $src1;",
 465   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rn_f>;
 466 def INT_NVVM_DIV_RZ_FTZ_F : F_MATH_2<"div.rz.ftz.f32 \t$dst, $src0, $src1;",
 467   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rz_ftz_f>;
 468 def INT_NVVM_DIV_RZ_F     : F_MATH_2<"div.rz.f32 \t$dst, $src0, $src1;",
 469   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rz_f>;
 470 def INT_NVVM_DIV_RM_FTZ_F : F_MATH_2<"div.rm.ftz.f32 \t$dst, $src0, $src1;",
 471   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rm_ftz_f>;
 472 def INT_NVVM_DIV_RM_F     : F_MATH_2<"div.rm.f32 \t$dst, $src0, $src1;",
 473   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rm_f>;
 474 def INT_NVVM_DIV_RP_FTZ_F : F_MATH_2<"div.rp.ftz.f32 \t$dst, $src0, $src1;",
 475   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rp_ftz_f>;
 476 def INT_NVVM_DIV_RP_F     : F_MATH_2<"div.rp.f32 \t$dst, $src0, $src1;",
 477   Float32Regs, Float32Regs, Float32Regs, int_nvvm_div_rp_f>;
 478
 479 def INT_NVVM_DIV_RN_D : F_MATH_2<"div.rn.f64 \t$dst, $src0, $src1;",
 480   Float64Regs, Float64Regs, Float64Regs, int_nvvm_div_rn_d>;
 481 def INT_NVVM_DIV_RZ_D : F_MATH_2<"div.rz.f64 \t$dst, $src0, $src1;",
 482   Float64Regs, Float64Regs, Float64Regs, int_nvvm_div_rz_d>;
 483 def INT_NVVM_DIV_RM_D : F_MATH_2<"div.rm.f64 \t$dst, $src0, $src1;",
 484   Float64Regs, Float64Regs, Float64Regs, int_nvvm_div_rm_d>;
 485 def INT_NVVM_DIV_RP_D : F_MATH_2<"div.rp.f64 \t$dst, $src0, $src1;",
 486   Float64Regs, Float64Regs, Float64Regs, int_nvvm_div_rp_d>;
 487
 488 //
 489 // Sad
 490 //
 491
 492 def INT_NVVM_SAD_I : F_MATH_3<"sad.s32 \t$dst, $src0, $src1, $src2;",
 493   Int32Regs, Int32Regs, Int32Regs, Int32Regs, int_nvvm_sad_i>;
 494 def INT_NVVM_SAD_UI : F_MATH_3<"sad.u32 \t$dst, $src0, $src1, $src2;",
 495   Int32Regs, Int32Regs, Int32Regs, Int32Regs, int_nvvm_sad_ui>;
 496
 497 //
 498 // Floor  Ceil
 499 //
 500
 501 def : Pat<(int_nvvm_floor_ftz_f Float32Regs:$a),
 502           (CVT_f32_f32 Float32Regs:$a, CvtRMI_FTZ)>;
 503 def : Pat<(int_nvvm_floor_f Float32Regs:$a),
 504           (CVT_f32_f32 Float32Regs:$a, CvtRMI)>;
 505 def : Pat<(int_nvvm_floor_d Float64Regs:$a),
 506           (CVT_f64_f64 Float64Regs:$a, CvtRMI)>;
 507
 508 def : Pat<(int_nvvm_ceil_ftz_f Float32Regs:$a),
 509           (CVT_f32_f32 Float32Regs:$a, CvtRPI_FTZ)>;
 510 def : Pat<(int_nvvm_ceil_f Float32Regs:$a),
 511           (CVT_f32_f32 Float32Regs:$a, CvtRPI)>;
 512 def : Pat<(int_nvvm_ceil_d Float64Regs:$a),
 513           (CVT_f64_f64 Float64Regs:$a, CvtRPI)>;
 514
 515 //
 516 // Abs
 517 //
 518
 519 def INT_NVVM_FABS_FTZ_F : F_MATH_1<"abs.ftz.f32 \t$dst, $src0;", Float32Regs,
 520   Float32Regs, int_nvvm_fabs_ftz_f>;
 521 def INT_NVVM_FABS_F : F_MATH_1<"abs.f32 \t$dst, $src0;", Float32Regs,
 522   Float32Regs, int_nvvm_fabs_f>;
 523
 524 def INT_NVVM_FABS_D : F_MATH_1<"abs.f64 \t$dst, $src0;", Float64Regs,
 525   Float64Regs, int_nvvm_fabs_d>;
 526
 527 //
 528 // Round
 529 //
 530
 531 def : Pat<(int_nvvm_round_ftz_f Float32Regs:$a),
 532           (CVT_f32_f32 Float32Regs:$a, CvtRNI_FTZ)>;
 533 def : Pat<(int_nvvm_round_f Float32Regs:$a),
 534           (CVT_f32_f32 Float32Regs:$a, CvtRNI)>;
 535 def : Pat<(int_nvvm_round_d Float64Regs:$a),
 536           (CVT_f64_f64 Float64Regs:$a, CvtRNI)>;
 537
 538 //
 539 // Trunc
 540 //
 541
 542 def : Pat<(int_nvvm_trunc_ftz_f Float32Regs:$a),
 543           (CVT_f32_f32 Float32Regs:$a, CvtRZI_FTZ)>;
 544 def : Pat<(int_nvvm_trunc_f Float32Regs:$a),
 545           (CVT_f32_f32 Float32Regs:$a, CvtRZI)>;
 546 def : Pat<(int_nvvm_trunc_d Float64Regs:$a),
 547           (CVT_f64_f64 Float64Regs:$a, CvtRZI)>;
 548
 549 //
 550 // Saturate
 551 //
 552
 553 def : Pat<(int_nvvm_saturate_ftz_f Float32Regs:$a),
 554           (CVT_f32_f32 Float32Regs:$a, CvtSAT_FTZ)>;
 555 def : Pat<(int_nvvm_saturate_f Float32Regs:$a),
 556           (CVT_f32_f32 Float32Regs:$a, CvtSAT)>;
 557 def : Pat<(int_nvvm_saturate_d Float64Regs:$a),
 558           (CVT_f64_f64 Float64Regs:$a, CvtSAT)>;
 559
 560 //
 561 // Exp2  Log2
 562 //
 563
 564 def INT_NVVM_EX2_APPROX_FTZ_F : F_MATH_1<"ex2.approx.ftz.f32 \t$dst, $src0;",
 565   Float32Regs, Float32Regs, int_nvvm_ex2_approx_ftz_f>;
 566 def INT_NVVM_EX2_APPROX_F : F_MATH_1<"ex2.approx.f32 \t$dst, $src0;",
 567   Float32Regs, Float32Regs, int_nvvm_ex2_approx_f>;
 568 def INT_NVVM_EX2_APPROX_D : F_MATH_1<"ex2.approx.f64 \t$dst, $src0;",
 569   Float64Regs, Float64Regs, int_nvvm_ex2_approx_d>;
 570
 571 def INT_NVVM_LG2_APPROX_FTZ_F : F_MATH_1<"lg2.approx.ftz.f32 \t$dst, $src0;",
 572   Float32Regs, Float32Regs, int_nvvm_lg2_approx_ftz_f>;
 573 def INT_NVVM_LG2_APPROX_F : F_MATH_1<"lg2.approx.f32 \t$dst, $src0;",
 574   Float32Regs, Float32Regs, int_nvvm_lg2_approx_f>;
 575 def INT_NVVM_LG2_APPROX_D : F_MATH_1<"lg2.approx.f64 \t$dst, $src0;",
 576   Float64Regs, Float64Regs, int_nvvm_lg2_approx_d>;
 577
 578 //
 579 // Sin  Cos
 580 //
 581
 582 def INT_NVVM_SIN_APPROX_FTZ_F : F_MATH_1<"sin.approx.ftz.f32 \t$dst, $src0;",
 583   Float32Regs, Float32Regs, int_nvvm_sin_approx_ftz_f>;
 584 def INT_NVVM_SIN_APPROX_F : F_MATH_1<"sin.approx.f32 \t$dst, $src0;",
 585   Float32Regs, Float32Regs, int_nvvm_sin_approx_f>;
 586
 587 def INT_NVVM_COS_APPROX_FTZ_F : F_MATH_1<"cos.approx.ftz.f32 \t$dst, $src0;",
 588   Float32Regs, Float32Regs, int_nvvm_cos_approx_ftz_f>;
 589 def INT_NVVM_COS_APPROX_F : F_MATH_1<"cos.approx.f32 \t$dst, $src0;",
 590   Float32Regs, Float32Regs, int_nvvm_cos_approx_f>;
 591
 592 //
 593 // Fma
 594 //
 595
 596 def INT_NVVM_FMA_RN_FTZ_F
 597   : F_MATH_3<"fma.rn.ftz.f32 \t$dst, $src0, $src1, $src2;", Float32Regs,
 598     Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rn_ftz_f>;
 599 def INT_NVVM_FMA_RN_F : F_MATH_3<"fma.rn.f32 \t$dst, $src0, $src1, $src2;",
 600   Float32Regs, Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rn_f>;
 601 def INT_NVVM_FMA_RZ_FTZ_F
 602   : F_MATH_3<"fma.rz.ftz.f32 \t$dst, $src0, $src1, $src2;", Float32Regs,
 603     Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rz_ftz_f>;
 604 def INT_NVVM_FMA_RZ_F : F_MATH_3<"fma.rz.f32 \t$dst, $src0, $src1, $src2;",
 605   Float32Regs, Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rz_f>;
 606 def INT_NVVM_FMA_RM_FTZ_F
 607   : F_MATH_3<"fma.rm.ftz.f32 \t$dst, $src0, $src1, $src2;", Float32Regs,
 608     Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rm_ftz_f>;
 609 def INT_NVVM_FMA_RM_F : F_MATH_3<"fma.rm.f32 \t$dst, $src0, $src1, $src2;",
 610   Float32Regs, Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rm_f>;
 611 def INT_NVVM_FMA_RP_FTZ_F
 612   : F_MATH_3<"fma.rp.ftz.f32 \t$dst, $src0, $src1, $src2;", Float32Regs,
 613     Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rp_ftz_f>;
 614 def INT_NVVM_FMA_RP_F : F_MATH_3<"fma.rp.f32 \t$dst, $src0, $src1, $src2;",
 615   Float32Regs, Float32Regs, Float32Regs, Float32Regs, int_nvvm_fma_rp_f>;
 616
 617 def INT_NVVM_FMA_RN_D : F_MATH_3<"fma.rn.f64 \t$dst, $src0, $src1, $src2;",
 618   Float64Regs, Float64Regs, Float64Regs, Float64Regs, int_nvvm_fma_rn_d>;
 619 def INT_NVVM_FMA_RZ_D : F_MATH_3<"fma.rz.f64 \t$dst, $src0, $src1, $src2;",
 620   Float64Regs, Float64Regs, Float64Regs, Float64Regs, int_nvvm_fma_rz_d>;
 621 def INT_NVVM_FMA_RM_D : F_MATH_3<"fma.rm.f64 \t$dst, $src0, $src1, $src2;",
 622   Float64Regs, Float64Regs, Float64Regs, Float64Regs, int_nvvm_fma_rm_d>;
 623 def INT_NVVM_FMA_RP_D : F_MATH_3<"fma.rp.f64 \t$dst, $src0, $src1, $src2;",
 624   Float64Regs, Float64Regs, Float64Regs, Float64Regs, int_nvvm_fma_rp_d>;
 625
 626 //
 627 // Rcp
 628 //
 629
 630 def INT_NVVM_RCP_RN_FTZ_F : F_MATH_1<"rcp.rn.ftz.f32 \t$dst, $src0;",
 631   Float32Regs, Float32Regs, int_nvvm_rcp_rn_ftz_f>;
 632 def INT_NVVM_RCP_RN_F : F_MATH_1<"rcp.rn.f32 \t$dst, $src0;",
 633   Float32Regs, Float32Regs, int_nvvm_rcp_rn_f>;
 634 def INT_NVVM_RCP_RZ_FTZ_F : F_MATH_1<"rcp.rz.ftz.f32 \t$dst, $src0;",
 635   Float32Regs, Float32Regs, int_nvvm_rcp_rz_ftz_f>;
 636 def INT_NVVM_RCP_RZ_F : F_MATH_1<"rcp.rz.f32 \t$dst, $src0;",
 637   Float32Regs, Float32Regs, int_nvvm_rcp_rz_f>;
 638 def INT_NVVM_RCP_RM_FTZ_F : F_MATH_1<"rcp.rm.ftz.f32 \t$dst, $src0;",
 639   Float32Regs, Float32Regs, int_nvvm_rcp_rm_ftz_f>;
 640 def INT_NVVM_RCP_RM_F : F_MATH_1<"rcp.rm.f32 \t$dst, $src0;",
 641   Float32Regs, Float32Regs, int_nvvm_rcp_rm_f>;
 642 def INT_NVVM_RCP_RP_FTZ_F : F_MATH_1<"rcp.rp.ftz.f32 \t$dst, $src0;",
 643   Float32Regs, Float32Regs, int_nvvm_rcp_rp_ftz_f>;
 644 def INT_NVVM_RCP_RP_F : F_MATH_1<"rcp.rp.f32 \t$dst, $src0;",
 645   Float32Regs, Float32Regs, int_nvvm_rcp_rp_f>;
 646
 647 def INT_NVVM_RCP_RN_D : F_MATH_1<"rcp.rn.f64 \t$dst, $src0;", Float64Regs,
 648   Float64Regs, int_nvvm_rcp_rn_d>;
 649 def INT_NVVM_RCP_RZ_D : F_MATH_1<"rcp.rz.f64 \t$dst, $src0;", Float64Regs,
 650   Float64Regs, int_nvvm_rcp_rz_d>;
 651 def INT_NVVM_RCP_RM_D : F_MATH_1<"rcp.rm.f64 \t$dst, $src0;", Float64Regs,
 652   Float64Regs, int_nvvm_rcp_rm_d>;
 653 def INT_NVVM_RCP_RP_D : F_MATH_1<"rcp.rp.f64 \t$dst, $src0;", Float64Regs,
 654   Float64Regs, int_nvvm_rcp_rp_d>;
 655
 656 def INT_NVVM_RCP_APPROX_FTZ_D : F_MATH_1<"rcp.approx.ftz.f64 \t$dst, $src0;",
 657   Float64Regs, Float64Regs, int_nvvm_rcp_approx_ftz_d>;
 658
 659 //
 660 // Sqrt
 661 //
 662
 663 def INT_NVVM_SQRT_RN_FTZ_F : F_MATH_1<"sqrt.rn.ftz.f32 \t$dst, $src0;",
 664   Float32Regs, Float32Regs, int_nvvm_sqrt_rn_ftz_f>;
 665 def INT_NVVM_SQRT_RN_F : F_MATH_1<"sqrt.rn.f32 \t$dst, $src0;", Float32Regs,
 666   Float32Regs, int_nvvm_sqrt_rn_f>;
 667 def INT_NVVM_SQRT_RZ_FTZ_F : F_MATH_1<"sqrt.rz.ftz.f32 \t$dst, $src0;",
 668   Float32Regs, Float32Regs, int_nvvm_sqrt_rz_ftz_f>;
 669 def INT_NVVM_SQRT_RZ_F : F_MATH_1<"sqrt.rz.f32 \t$dst, $src0;", Float32Regs,
 670   Float32Regs, int_nvvm_sqrt_rz_f>;
 671 def INT_NVVM_SQRT_RM_FTZ_F : F_MATH_1<"sqrt.rm.ftz.f32 \t$dst, $src0;",
 672   Float32Regs, Float32Regs, int_nvvm_sqrt_rm_ftz_f>;
 673 def INT_NVVM_SQRT_RM_F : F_MATH_1<"sqrt.rm.f32 \t$dst, $src0;", Float32Regs,
 674   Float32Regs, int_nvvm_sqrt_rm_f>;
 675 def INT_NVVM_SQRT_RP_FTZ_F : F_MATH_1<"sqrt.rp.ftz.f32 \t$dst, $src0;",
 676   Float32Regs, Float32Regs, int_nvvm_sqrt_rp_ftz_f>;
 677 def INT_NVVM_SQRT_RP_F : F_MATH_1<"sqrt.rp.f32 \t$dst, $src0;", Float32Regs,
 678   Float32Regs, int_nvvm_sqrt_rp_f>;
 679 def INT_NVVM_SQRT_APPROX_FTZ_F : F_MATH_1<"sqrt.approx.ftz.f32 \t$dst, $src0;",
 680   Float32Regs, Float32Regs, int_nvvm_sqrt_approx_ftz_f>;
 681 def INT_NVVM_SQRT_APPROX_F : F_MATH_1<"sqrt.approx.f32 \t$dst, $src0;",
 682   Float32Regs, Float32Regs, int_nvvm_sqrt_approx_f>;
 683
 684 def INT_NVVM_SQRT_RN_D : F_MATH_1<"sqrt.rn.f64 \t$dst, $src0;", Float64Regs,
 685   Float64Regs, int_nvvm_sqrt_rn_d>;
 686 def INT_NVVM_SQRT_RZ_D : F_MATH_1<"sqrt.rz.f64 \t$dst, $src0;", Float64Regs,
 687   Float64Regs, int_nvvm_sqrt_rz_d>;
 688 def INT_NVVM_SQRT_RM_D : F_MATH_1<"sqrt.rm.f64 \t$dst, $src0;", Float64Regs,
 689   Float64Regs, int_nvvm_sqrt_rm_d>;
 690 def INT_NVVM_SQRT_RP_D : F_MATH_1<"sqrt.rp.f64 \t$dst, $src0;", Float64Regs,
 691   Float64Regs, int_nvvm_sqrt_rp_d>;
 692
 693 // nvvm_sqrt intrinsic
 694 def : Pat<(int_nvvm_sqrt_f Float32Regs:$a),
 695           (INT_NVVM_SQRT_RN_FTZ_F Float32Regs:$a)>, Requires<[doF32FTZ, do_SQRTF32_RN]>;
 696 def : Pat<(int_nvvm_sqrt_f Float32Regs:$a),
 697           (INT_NVVM_SQRT_RN_F Float32Regs:$a)>, Requires<[do_SQRTF32_RN]>;
 698 def : Pat<(int_nvvm_sqrt_f Float32Regs:$a),
 699           (INT_NVVM_SQRT_APPROX_FTZ_F Float32Regs:$a)>, Requires<[doF32FTZ]>;
 700 def : Pat<(int_nvvm_sqrt_f Float32Regs:$a),
 701           (INT_NVVM_SQRT_APPROX_F Float32Regs:$a)>;
 702
 703 //
 704 // Rsqrt
 705 //
 706
 707 def INT_NVVM_RSQRT_APPROX_FTZ_F
 708   : F_MATH_1<"rsqrt.approx.ftz.f32 \t$dst, $src0;", Float32Regs, Float32Regs,
 709     int_nvvm_rsqrt_approx_ftz_f>;
 710 def INT_NVVM_RSQRT_APPROX_F : F_MATH_1<"rsqrt.approx.f32 \t$dst, $src0;",
 711   Float32Regs, Float32Regs, int_nvvm_rsqrt_approx_f>;
 712 def INT_NVVM_RSQRT_APPROX_D : F_MATH_1<"rsqrt.approx.f64 \t$dst, $src0;",
 713   Float64Regs, Float64Regs, int_nvvm_rsqrt_approx_d>;
 714
 715 //
 716 // Add
 717 //
 718
 719 def INT_NVVM_ADD_RN_FTZ_F : F_MATH_2<"add.rn.ftz.f32 \t$dst, $src0, $src1;",
 720   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rn_ftz_f>;
 721 def INT_NVVM_ADD_RN_F : F_MATH_2<"add.rn.f32 \t$dst, $src0, $src1;",
 722   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rn_f>;
 723 def INT_NVVM_ADD_RZ_FTZ_F : F_MATH_2<"add.rz.ftz.f32 \t$dst, $src0, $src1;",
 724   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rz_ftz_f>;
 725 def INT_NVVM_ADD_RZ_F : F_MATH_2<"add.rz.f32 \t$dst, $src0, $src1;",
 726   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rz_f>;
 727 def INT_NVVM_ADD_RM_FTZ_F : F_MATH_2<"add.rm.ftz.f32 \t$dst, $src0, $src1;",
 728   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rm_ftz_f>;
 729 def INT_NVVM_ADD_RM_F : F_MATH_2<"add.rm.f32 \t$dst, $src0, $src1;",
 730   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rm_f>;
 731 def INT_NVVM_ADD_RP_FTZ_F : F_MATH_2<"add.rp.ftz.f32 \t$dst, $src0, $src1;",
 732   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rp_ftz_f>;
 733 def INT_NVVM_ADD_RP_F : F_MATH_2<"add.rp.f32 \t$dst, $src0, $src1;",
 734   Float32Regs, Float32Regs, Float32Regs, int_nvvm_add_rp_f>;
 735
 736 def INT_NVVM_ADD_RN_D : F_MATH_2<"add.rn.f64 \t$dst, $src0, $src1;",
 737   Float64Regs, Float64Regs, Float64Regs, int_nvvm_add_rn_d>;
 738 def INT_NVVM_ADD_RZ_D : F_MATH_2<"add.rz.f64 \t$dst, $src0, $src1;",
 739   Float64Regs, Float64Regs, Float64Regs, int_nvvm_add_rz_d>;
 740 def INT_NVVM_ADD_RM_D : F_MATH_2<"add.rm.f64 \t$dst, $src0, $src1;",
 741   Float64Regs, Float64Regs, Float64Regs, int_nvvm_add_rm_d>;
 742 def INT_NVVM_ADD_RP_D : F_MATH_2<"add.rp.f64 \t$dst, $src0, $src1;",
 743   Float64Regs, Float64Regs, Float64Regs, int_nvvm_add_rp_d>;
 744
 745 //
 746 // Convert
 747 //
 748
 749 def : Pat<(int_nvvm_d2f_rn_ftz Float64Regs:$a),
 750           (CVT_f32_f64 Float64Regs:$a, CvtRN_FTZ)>;
 751 def : Pat<(int_nvvm_d2f_rn Float64Regs:$a),
 752           (CVT_f32_f64 Float64Regs:$a, CvtRN)>;
 753 def : Pat<(int_nvvm_d2f_rz_ftz Float64Regs:$a),
 754           (CVT_f32_f64 Float64Regs:$a, CvtRZ_FTZ)>;
 755 def : Pat<(int_nvvm_d2f_rz Float64Regs:$a),
 756           (CVT_f32_f64 Float64Regs:$a, CvtRZ)>;
 757 def : Pat<(int_nvvm_d2f_rm_ftz Float64Regs:$a),
 758           (CVT_f32_f64 Float64Regs:$a, CvtRM_FTZ)>;
 759 def : Pat<(int_nvvm_d2f_rm Float64Regs:$a),
 760           (CVT_f32_f64 Float64Regs:$a, CvtRM)>;
 761 def : Pat<(int_nvvm_d2f_rp_ftz Float64Regs:$a),
 762           (CVT_f32_f64 Float64Regs:$a, CvtRP_FTZ)>;
 763 def : Pat<(int_nvvm_d2f_rp Float64Regs:$a),
 764           (CVT_f32_f64 Float64Regs:$a, CvtRP)>;
 765
 766 def : Pat<(int_nvvm_d2i_rn Float64Regs:$a),
 767           (CVT_s32_f64 Float64Regs:$a, CvtRNI)>;
 768 def : Pat<(int_nvvm_d2i_rz Float64Regs:$a),
 769           (CVT_s32_f64 Float64Regs:$a, CvtRZI)>;
 770 def : Pat<(int_nvvm_d2i_rm Float64Regs:$a),
 771           (CVT_s32_f64 Float64Regs:$a, CvtRMI)>;
 772 def : Pat<(int_nvvm_d2i_rp Float64Regs:$a),
 773           (CVT_s32_f64 Float64Regs:$a, CvtRPI)>;
 774
 775 def : Pat<(int_nvvm_d2ui_rn Float64Regs:$a),
 776           (CVT_u32_f64 Float64Regs:$a, CvtRNI)>;
 777 def : Pat<(int_nvvm_d2ui_rz Float64Regs:$a),
 778           (CVT_u32_f64 Float64Regs:$a, CvtRZI)>;
 779 def : Pat<(int_nvvm_d2ui_rm Float64Regs:$a),
 780           (CVT_u32_f64 Float64Regs:$a, CvtRMI)>;
 781 def : Pat<(int_nvvm_d2ui_rp Float64Regs:$a),
 782           (CVT_u32_f64 Float64Regs:$a, CvtRPI)>;
 783
 784 def : Pat<(int_nvvm_i2d_rn Int32Regs:$a),
 785           (CVT_f64_s32 Int32Regs:$a, CvtRN)>;
 786 def : Pat<(int_nvvm_i2d_rz Int32Regs:$a),
 787           (CVT_f64_s32 Int32Regs:$a, CvtRZ)>;
 788 def : Pat<(int_nvvm_i2d_rm Int32Regs:$a),
 789           (CVT_f64_s32 Int32Regs:$a, CvtRM)>;
 790 def : Pat<(int_nvvm_i2d_rp Int32Regs:$a),
 791           (CVT_f64_s32 Int32Regs:$a, CvtRP)>;
 792
 793 def : Pat<(int_nvvm_ui2d_rn Int32Regs:$a),
 794           (CVT_f64_u32 Int32Regs:$a, CvtRN)>;
 795 def : Pat<(int_nvvm_ui2d_rz Int32Regs:$a),
 796           (CVT_f64_u32 Int32Regs:$a, CvtRZ)>;
 797 def : Pat<(int_nvvm_ui2d_rm Int32Regs:$a),
 798           (CVT_f64_u32 Int32Regs:$a, CvtRM)>;
 799 def : Pat<(int_nvvm_ui2d_rp Int32Regs:$a),
 800           (CVT_f64_u32 Int32Regs:$a, CvtRP)>;
 801
 802 def : Pat<(int_nvvm_f2i_rn_ftz Float32Regs:$a),
 803           (CVT_s32_f32 Float32Regs:$a, CvtRNI_FTZ)>;
 804 def : Pat<(int_nvvm_f2i_rn Float32Regs:$a),
 805           (CVT_s32_f32 Float32Regs:$a, CvtRNI)>;
 806 def : Pat<(int_nvvm_f2i_rz_ftz Float32Regs:$a),
 807           (CVT_s32_f32 Float32Regs:$a, CvtRZI_FTZ)>;
 808 def : Pat<(int_nvvm_f2i_rz Float32Regs:$a),
 809           (CVT_s32_f32 Float32Regs:$a, CvtRZI)>;
 810 def : Pat<(int_nvvm_f2i_rm_ftz Float32Regs:$a),
 811           (CVT_s32_f32 Float32Regs:$a, CvtRMI_FTZ)>;
 812 def : Pat<(int_nvvm_f2i_rm Float32Regs:$a),
 813           (CVT_s32_f32 Float32Regs:$a, CvtRMI)>;
 814 def : Pat<(int_nvvm_f2i_rp_ftz Float32Regs:$a),
 815           (CVT_s32_f32 Float32Regs:$a, CvtRPI_FTZ)>;
 816 def : Pat<(int_nvvm_f2i_rp Float32Regs:$a),
 817           (CVT_s32_f32 Float32Regs:$a, CvtRPI)>;
 818
 819 def : Pat<(int_nvvm_f2ui_rn_ftz Float32Regs:$a),
 820           (CVT_u32_f32 Float32Regs:$a, CvtRNI_FTZ)>;
 821 def : Pat<(int_nvvm_f2ui_rn Float32Regs:$a),
 822           (CVT_u32_f32 Float32Regs:$a, CvtRNI)>;
 823 def : Pat<(int_nvvm_f2ui_rz_ftz Float32Regs:$a),
 824           (CVT_u32_f32 Float32Regs:$a, CvtRZI_FTZ)>;
 825 def : Pat<(int_nvvm_f2ui_rz Float32Regs:$a),
 826           (CVT_u32_f32 Float32Regs:$a, CvtRZI)>;
 827 def : Pat<(int_nvvm_f2ui_rm_ftz Float32Regs:$a),
 828           (CVT_u32_f32 Float32Regs:$a, CvtRMI_FTZ)>;
 829 def : Pat<(int_nvvm_f2ui_rm Float32Regs:$a),
 830           (CVT_u32_f32 Float32Regs:$a, CvtRMI)>;
 831 def : Pat<(int_nvvm_f2ui_rp_ftz Float32Regs:$a),
 832           (CVT_u32_f32 Float32Regs:$a, CvtRPI_FTZ)>;
 833 def : Pat<(int_nvvm_f2ui_rp Float32Regs:$a),
 834           (CVT_u32_f32 Float32Regs:$a, CvtRPI)>;
 835
 836 def : Pat<(int_nvvm_i2f_rn Int32Regs:$a),
 837           (CVT_f32_s32 Int32Regs:$a, CvtRN)>;
 838 def : Pat<(int_nvvm_i2f_rz Int32Regs:$a),
 839           (CVT_f32_s32 Int32Regs:$a, CvtRZ)>;
 840 def : Pat<(int_nvvm_i2f_rm Int32Regs:$a),
 841           (CVT_f32_s32 Int32Regs:$a, CvtRM)>;
 842 def : Pat<(int_nvvm_i2f_rp Int32Regs:$a),
 843           (CVT_f32_s32 Int32Regs:$a, CvtRP)>;
 844
 845 def : Pat<(int_nvvm_ui2f_rn Int32Regs:$a),
 846           (CVT_f32_u32 Int32Regs:$a, CvtRN)>;
 847 def : Pat<(int_nvvm_ui2f_rz Int32Regs:$a),
 848           (CVT_f32_u32 Int32Regs:$a, CvtRZ)>;
 849 def : Pat<(int_nvvm_ui2f_rm Int32Regs:$a),
 850           (CVT_f32_u32 Int32Regs:$a, CvtRM)>;
 851 def : Pat<(int_nvvm_ui2f_rp Int32Regs:$a),
 852           (CVT_f32_u32 Int32Regs:$a, CvtRP)>;
 853
 854 def INT_NVVM_LOHI_I2D : F_MATH_2<"mov.b64 \t$dst, {{$src0, $src1}};",
 855   Float64Regs, Int32Regs, Int32Regs, int_nvvm_lohi_i2d>;
 856
 857 def INT_NVVM_D2I_LO : F_MATH_1<
 858   !strconcat("{{\n\t",
 859              ".reg .b32 %temp; \n\t",
 860              "mov.b64 \t{$dst, %temp}, $src0;\n\t",
 861              "}}"),
 862   Int32Regs, Float64Regs, int_nvvm_d2i_lo>;
 863 def INT_NVVM_D2I_HI : F_MATH_1<
 864   !strconcat("{{\n\t",
 865              ".reg .b32 %temp; \n\t",
 866              "mov.b64 \t{%temp, $dst}, $src0;\n\t",
 867              "}}"),
 868   Int32Regs, Float64Regs, int_nvvm_d2i_hi>;
 869
 870 def : Pat<(int_nvvm_f2ll_rn_ftz Float32Regs:$a),
 871           (CVT_s64_f32 Float32Regs:$a, CvtRNI_FTZ)>;
 872 def : Pat<(int_nvvm_f2ll_rn Float32Regs:$a),
 873           (CVT_s64_f32 Float32Regs:$a, CvtRNI)>;
 874 def : Pat<(int_nvvm_f2ll_rz_ftz Float32Regs:$a),
 875           (CVT_s64_f32 Float32Regs:$a, CvtRZI_FTZ)>;
 876 def : Pat<(int_nvvm_f2ll_rz Float32Regs:$a),
 877           (CVT_s64_f32 Float32Regs:$a, CvtRZI)>;
 878 def : Pat<(int_nvvm_f2ll_rm_ftz Float32Regs:$a),
 879           (CVT_s64_f32 Float32Regs:$a, CvtRMI_FTZ)>;
 880 def : Pat<(int_nvvm_f2ll_rm Float32Regs:$a),
 881           (CVT_s64_f32 Float32Regs:$a, CvtRMI)>;
 882 def : Pat<(int_nvvm_f2ll_rp_ftz Float32Regs:$a),
 883           (CVT_s64_f32 Float32Regs:$a, CvtRPI_FTZ)>;
 884 def : Pat<(int_nvvm_f2ll_rp Float32Regs:$a),
 885           (CVT_s64_f32 Float32Regs:$a, CvtRPI)>;
 886
 887 def : Pat<(int_nvvm_f2ull_rn_ftz Float32Regs:$a),
 888           (CVT_u64_f32 Float32Regs:$a, CvtRNI_FTZ)>;
 889 def : Pat<(int_nvvm_f2ull_rn Float32Regs:$a),
 890           (CVT_u64_f32 Float32Regs:$a, CvtRNI)>;
 891 def : Pat<(int_nvvm_f2ull_rz_ftz Float32Regs:$a),
 892           (CVT_u64_f32 Float32Regs:$a, CvtRZI_FTZ)>;
 893 def : Pat<(int_nvvm_f2ull_rz Float32Regs:$a),
 894           (CVT_u64_f32 Float32Regs:$a, CvtRZI)>;
 895 def : Pat<(int_nvvm_f2ull_rm_ftz Float32Regs:$a),
 896           (CVT_u64_f32 Float32Regs:$a, CvtRMI_FTZ)>;
 897 def : Pat<(int_nvvm_f2ull_rm Float32Regs:$a),
 898           (CVT_u64_f32 Float32Regs:$a, CvtRMI)>;
 899 def : Pat<(int_nvvm_f2ull_rp_ftz Float32Regs:$a),
 900           (CVT_u64_f32 Float32Regs:$a, CvtRPI_FTZ)>;
 901 def : Pat<(int_nvvm_f2ull_rp Float32Regs:$a),
 902           (CVT_u64_f32 Float32Regs:$a, CvtRPI)>;
 903
 904 def : Pat<(int_nvvm_d2ll_rn Float64Regs:$a),
 905           (CVT_s64_f64 Float64Regs:$a, CvtRNI)>;
 906 def : Pat<(int_nvvm_d2ll_rz Float64Regs:$a),
 907           (CVT_s64_f64 Float64Regs:$a, CvtRZI)>;
 908 def : Pat<(int_nvvm_d2ll_rm Float64Regs:$a),
 909           (CVT_s64_f64 Float64Regs:$a, CvtRMI)>;
 910 def : Pat<(int_nvvm_d2ll_rp Float64Regs:$a),
 911           (CVT_s64_f64 Float64Regs:$a, CvtRPI)>;
 912
 913 def : Pat<(int_nvvm_d2ull_rn Float64Regs:$a),
 914           (CVT_u64_f64 Float64Regs:$a, CvtRNI)>;
 915 def : Pat<(int_nvvm_d2ull_rz Float64Regs:$a),
 916           (CVT_u64_f64 Float64Regs:$a, CvtRZI)>;
 917 def : Pat<(int_nvvm_d2ull_rm Float64Regs:$a),
 918           (CVT_u64_f64 Float64Regs:$a, CvtRMI)>;
 919 def : Pat<(int_nvvm_d2ull_rp Float64Regs:$a),
 920           (CVT_u64_f64 Float64Regs:$a, CvtRPI)>;
 921
 922 def : Pat<(int_nvvm_ll2f_rn Int64Regs:$a),
 923           (CVT_f32_s64 Int64Regs:$a, CvtRN)>;
 924 def : Pat<(int_nvvm_ll2f_rz Int64Regs:$a),
 925           (CVT_f32_s64 Int64Regs:$a, CvtRZ)>;
 926 def : Pat<(int_nvvm_ll2f_rm Int64Regs:$a),
 927           (CVT_f32_s64 Int64Regs:$a, CvtRM)>;
 928 def : Pat<(int_nvvm_ll2f_rp Int64Regs:$a),
 929           (CVT_f32_s64 Int64Regs:$a, CvtRP)>;
 930
 931 def : Pat<(int_nvvm_ull2f_rn Int64Regs:$a),
 932           (CVT_f32_u64 Int64Regs:$a, CvtRN)>;
 933 def : Pat<(int_nvvm_ull2f_rz Int64Regs:$a),
 934           (CVT_f32_u64 Int64Regs:$a, CvtRZ)>;
 935 def : Pat<(int_nvvm_ull2f_rm Int64Regs:$a),
 936           (CVT_f32_u64 Int64Regs:$a, CvtRM)>;
 937 def : Pat<(int_nvvm_ull2f_rp Int64Regs:$a),
 938           (CVT_f32_u64 Int64Regs:$a, CvtRP)>;
 939
 940 def : Pat<(int_nvvm_ll2d_rn Int64Regs:$a),
 941           (CVT_f64_s64 Int64Regs:$a, CvtRN)>;
 942 def : Pat<(int_nvvm_ll2d_rz Int64Regs:$a),
 943           (CVT_f64_s64 Int64Regs:$a, CvtRZ)>;
 944 def : Pat<(int_nvvm_ll2d_rm Int64Regs:$a),
 945           (CVT_f64_s64 Int64Regs:$a, CvtRM)>;
 946 def : Pat<(int_nvvm_ll2d_rp Int64Regs:$a),
 947           (CVT_f64_s64 Int64Regs:$a, CvtRP)>;
 948
 949 def : Pat<(int_nvvm_ull2d_rn Int64Regs:$a),
 950           (CVT_f64_u64 Int64Regs:$a, CvtRN)>;
 951 def : Pat<(int_nvvm_ull2d_rz Int64Regs:$a),
 952           (CVT_f64_u64 Int64Regs:$a, CvtRZ)>;
 953 def : Pat<(int_nvvm_ull2d_rm Int64Regs:$a),
 954           (CVT_f64_u64 Int64Regs:$a, CvtRM)>;
 955 def : Pat<(int_nvvm_ull2d_rp Int64Regs:$a),
 956           (CVT_f64_u64 Int64Regs:$a, CvtRP)>;
 957
 958
 959 def : Pat<(int_nvvm_f2h_rn_ftz Float32Regs:$a),
 960           (BITCONVERT_16_F2I (CVT_f16_f32 Float32Regs:$a, CvtRN_FTZ))>;
 961 def : Pat<(int_nvvm_f2h_rn Float32Regs:$a),
 962           (BITCONVERT_16_F2I (CVT_f16_f32 Float32Regs:$a, CvtRN))>;
 963
 964 //
 965 // Bitcast
 966 //
 967
 968 def INT_NVVM_BITCAST_F2I : F_MATH_1<"mov.b32 \t$dst, $src0;", Int32Regs,
 969   Float32Regs, int_nvvm_bitcast_f2i>;
 970 def INT_NVVM_BITCAST_I2F : F_MATH_1<"mov.b32 \t$dst, $src0;", Float32Regs,
 971   Int32Regs, int_nvvm_bitcast_i2f>;
 972
 973 def INT_NVVM_BITCAST_LL2D : F_MATH_1<"mov.b64 \t$dst, $src0;", Float64Regs,
 974   Int64Regs, int_nvvm_bitcast_ll2d>;
 975 def INT_NVVM_BITCAST_D2LL : F_MATH_1<"mov.b64 \t$dst, $src0;", Int64Regs,
 976   Float64Regs, int_nvvm_bitcast_d2ll>;
 977
 978 //
 979 // FNS
 980 //
 981
 982 class INT_FNS_MBO<dag ins, dag Operands>
 983   : NVPTXInst<(outs Int32Regs:$dst), ins,
 984                "fns.b32 \t$dst, $mask, $base, $offset;",
 985                [(set Int32Regs:$dst, Operands )]>,
 986     Requires<[hasPTX60, hasSM30]>;
 987
 988 def INT_FNS_rrr : INT_FNS_MBO<(ins Int32Regs:$mask, Int32Regs:$base, Int32Regs:$offset),
 989                      (int_nvvm_fns Int32Regs:$mask, Int32Regs:$base, Int32Regs:$offset)>;
 990 def INT_FNS_rri : INT_FNS_MBO<(ins Int32Regs:$mask, Int32Regs:$base,    i32imm:$offset),
 991                      (int_nvvm_fns Int32Regs:$mask, Int32Regs:$base,       imm:$offset)>;
 992 def INT_FNS_rir : INT_FNS_MBO<(ins Int32Regs:$mask,    i32imm:$base, Int32Regs:$offset),
 993                      (int_nvvm_fns Int32Regs:$mask,       imm:$base, Int32Regs:$offset)>;
 994 def INT_FNS_rii : INT_FNS_MBO<(ins Int32Regs:$mask,    i32imm:$base,    i32imm:$offset),
 995                      (int_nvvm_fns Int32Regs:$mask,       imm:$base,       imm:$offset)>;
 996 def INT_FNS_irr : INT_FNS_MBO<(ins    i32imm:$mask, Int32Regs:$base, Int32Regs:$offset),
 997                      (int_nvvm_fns       imm:$mask, Int32Regs:$base, Int32Regs:$offset)>;
 998 def INT_FNS_iri : INT_FNS_MBO<(ins    i32imm:$mask, Int32Regs:$base,    i32imm:$offset),
 999                      (int_nvvm_fns       imm:$mask, Int32Regs:$base,       imm:$offset)>;
1000 def INT_FNS_iir : INT_FNS_MBO<(ins    i32imm:$mask,    i32imm:$base, Int32Regs:$offset),
1001                      (int_nvvm_fns       imm:$mask,       imm:$base, Int32Regs:$offset)>;
1002 def INT_FNS_iii : INT_FNS_MBO<(ins    i32imm:$mask,    i32imm:$base,    i32imm:$offset),
1003                      (int_nvvm_fns       imm:$mask,       imm:$base,       imm:$offset)>;
1004
1005 //-----------------------------------
1006 // Atomic Functions
1007 //-----------------------------------
1008
1009 class ATOMIC_GLOBAL_CHK <dag ops, dag frag>
1010  : PatFrag<ops, frag, [{
1011    return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_GLOBAL);
1012 }]>;
1013 class ATOMIC_SHARED_CHK <dag ops, dag frag>
1014  : PatFrag<ops, frag, [{
1015    return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_SHARED);
1016 }]>;
1017 class ATOMIC_GENERIC_CHK <dag ops, dag frag>
1018  : PatFrag<ops, frag, [{
1019    return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_GENERIC);
1020 }]>;
1021
1022 multiclass F_ATOMIC_2_imp<NVPTXRegClass ptrclass, NVPTXRegClass regclass,
1023   string SpaceStr, string TypeStr, string OpcStr, PatFrag IntOp,
1024   Operand IMMType, SDNode IMM, list<Predicate> Pred> {
1025   def reg : NVPTXInst<(outs regclass:$dst), (ins ptrclass:$addr, regclass:$b),
1026     !strconcat("atom", SpaceStr, OpcStr, TypeStr, " \t$dst, [$addr], $b;"),
1027     [(set regclass:$dst, (IntOp ptrclass:$addr, regclass:$b))]>,
1028   Requires<Pred>;
1029   def imm : NVPTXInst<(outs regclass:$dst), (ins ptrclass:$addr, IMMType:$b),
1030     !strconcat("atom", SpaceStr, OpcStr, TypeStr, " \t$dst, [$addr], $b;", ""),
1031     [(set regclass:$dst, (IntOp ptrclass:$addr, IMM:$b))]>,
1032   Requires<Pred>;
1033 }
1034 multiclass F_ATOMIC_2<NVPTXRegClass regclass, string SpaceStr, string TypeStr,
1035   string OpcStr, PatFrag IntOp, Operand IMMType, SDNode IMM,
1036   list<Predicate> Pred = []> {
1037   defm p32 : F_ATOMIC_2_imp<Int32Regs, regclass, SpaceStr, TypeStr, OpcStr,
1038     IntOp, IMMType, IMM, Pred>;
1039   defm p64 : F_ATOMIC_2_imp<Int64Regs, regclass, SpaceStr, TypeStr, OpcStr,
1040     IntOp, IMMType, IMM, Pred>;
1041 }
1042
1043 // has 2 operands, neg the second one
1044 multiclass F_ATOMIC_2_NEG_imp<NVPTXRegClass ptrclass, NVPTXRegClass regclass,
1045   string SpaceStr, string TypeStr, string OpcStr, PatFrag IntOp,
1046   Operand IMMType, list<Predicate> Pred> {
1047   def reg : NVPTXInst<(outs regclass:$dst), (ins ptrclass:$addr, regclass:$b),
1048     !strconcat(
1049       "{{ \n\t",
1050       ".reg \t.s", TypeStr, " temp; \n\t",
1051       "neg.s", TypeStr, " \ttemp, $b; \n\t",
1052       "atom", SpaceStr, OpcStr, ".u", TypeStr, " \t$dst, [$addr], temp; \n\t",
1053       "}}"),
1054     [(set regclass:$dst, (IntOp ptrclass:$addr, regclass:$b))]>,
1055   Requires<Pred>;
1056 }
1057 multiclass F_ATOMIC_2_NEG<NVPTXRegClass regclass, string SpaceStr,
1058   string TypeStr, string OpcStr, PatFrag IntOp, Operand IMMType,
1059   list<Predicate> Pred = []> {
1060  defm p32: F_ATOMIC_2_NEG_imp<Int32Regs, regclass, SpaceStr, TypeStr, OpcStr,
1061    IntOp, IMMType, Pred> ;
1062  defm p64: F_ATOMIC_2_NEG_imp<Int64Regs, regclass, SpaceStr, TypeStr, OpcStr,
1063    IntOp, IMMType, Pred> ;
1064 }
1065
1066 // has 3 operands
1067 multiclass F_ATOMIC_3_imp<NVPTXRegClass ptrclass, NVPTXRegClass regclass,
1068   string SpaceStr, string TypeStr, string OpcStr, PatFrag IntOp,
1069   Operand IMMType, list<Predicate> Pred> {
1070   def reg : NVPTXInst<(outs regclass:$dst),
1071     (ins ptrclass:$addr, regclass:$b, regclass:$c),
1072     !strconcat("atom", SpaceStr, OpcStr, TypeStr, " \t$dst, [$addr], $b, $c;"),
1073     [(set regclass:$dst, (IntOp ptrclass:$addr, regclass:$b, regclass:$c))]>,
1074   Requires<Pred>;
1075
1076   def imm1 : NVPTXInst<(outs regclass:$dst),
1077     (ins ptrclass:$addr, IMMType:$b, regclass:$c),
1078     !strconcat("atom", SpaceStr, OpcStr, TypeStr, " \t$dst, [$addr], $b, $c;"),
1079     [(set regclass:$dst, (IntOp ptrclass:$addr, imm:$b, regclass:$c))]>,
1080   Requires<Pred>;
1081
1082   def imm2 : NVPTXInst<(outs regclass:$dst),
1083     (ins ptrclass:$addr, regclass:$b, IMMType:$c),
1084     !strconcat("atom", SpaceStr, OpcStr, TypeStr, " \t$dst, [$addr], $b, $c;", ""),
1085     [(set regclass:$dst, (IntOp ptrclass:$addr, regclass:$b, imm:$c))]>,
1086   Requires<Pred>;
1087
1088   def imm3 : NVPTXInst<(outs regclass:$dst),
1089     (ins ptrclass:$addr, IMMType:$b, IMMType:$c),
1090     !strconcat("atom", SpaceStr, OpcStr, TypeStr, " \t$dst, [$addr], $b, $c;"),
1091     [(set regclass:$dst, (IntOp ptrclass:$addr, imm:$b, imm:$c))]>,
1092   Requires<Pred>;
1093 }
1094 multiclass F_ATOMIC_3<NVPTXRegClass regclass, string SpaceStr, string TypeStr,
1095   string OpcStr, PatFrag IntOp, Operand IMMType, list<Predicate> Pred = []> {
1096   defm p32 : F_ATOMIC_3_imp<Int32Regs, regclass, SpaceStr, TypeStr, OpcStr,
1097     IntOp, IMMType, Pred>;
1098   defm p64 : F_ATOMIC_3_imp<Int64Regs, regclass, SpaceStr, TypeStr, OpcStr,
1099     IntOp, IMMType, Pred>;
1100 }
1101
1102 // atom_add
1103
1104 def atomic_load_add_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1105   (atomic_load_add_32 node:$a, node:$b)>;
1106 def atomic_load_add_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1107   (atomic_load_add_32 node:$a, node:$b)>;
1108 def atomic_load_add_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1109   (atomic_load_add_32 node:$a, node:$b)>;
1110 def atomic_load_add_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1111   (atomic_load_add_64 node:$a, node:$b)>;
1112 def atomic_load_add_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1113   (atomic_load_add_64 node:$a, node:$b)>;
1114 def atomic_load_add_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1115   (atomic_load_add_64 node:$a, node:$b)>;
1116 def atomic_load_add_f32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1117   (int_nvvm_atomic_load_add_f32 node:$a, node:$b)>;
1118 def atomic_load_add_f32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1119   (int_nvvm_atomic_load_add_f32 node:$a, node:$b)>;
1120 def atomic_load_add_f32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1121   (int_nvvm_atomic_load_add_f32 node:$a, node:$b)>;
1122 def atomic_load_add_f64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1123   (int_nvvm_atomic_load_add_f64 node:$a, node:$b)>;
1124 def atomic_load_add_f64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1125   (int_nvvm_atomic_load_add_f64 node:$a, node:$b)>;
1126 def atomic_load_add_f64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1127   (int_nvvm_atomic_load_add_f64 node:$a, node:$b)>;
1128
1129 defm INT_PTX_ATOM_ADD_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".u32", ".add",
1130   atomic_load_add_32_g, i32imm, imm>;
1131 defm INT_PTX_ATOM_ADD_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".u32", ".add",
1132   atomic_load_add_32_s, i32imm, imm>;
1133 defm INT_PTX_ATOM_ADD_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".u32", ".add",
1134   atomic_load_add_32_gen, i32imm, imm>;
1135 defm INT_PTX_ATOM_ADD_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".u32",
1136   ".add", atomic_load_add_32_gen, i32imm, imm>;
1137
1138 defm INT_PTX_ATOM_ADD_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".u64", ".add",
1139   atomic_load_add_64_g, i64imm, imm>;
1140 defm INT_PTX_ATOM_ADD_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".u64", ".add",
1141   atomic_load_add_64_s, i64imm, imm>;
1142 defm INT_PTX_ATOM_ADD_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".u64", ".add",
1143   atomic_load_add_64_gen, i64imm, imm>;
1144 defm INT_PTX_ATOM_ADD_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global", ".u64",
1145   ".add", atomic_load_add_64_gen, i64imm, imm>;
1146
1147 defm INT_PTX_ATOM_ADD_G_F32 : F_ATOMIC_2<Float32Regs, ".global", ".f32", ".add",
1148   atomic_load_add_f32_g, f32imm, fpimm>;
1149 defm INT_PTX_ATOM_ADD_S_F32 : F_ATOMIC_2<Float32Regs, ".shared", ".f32", ".add",
1150   atomic_load_add_f32_s, f32imm, fpimm>;
1151 defm INT_PTX_ATOM_ADD_GEN_F32 : F_ATOMIC_2<Float32Regs, "", ".f32", ".add",
1152   atomic_load_add_f32_gen, f32imm, fpimm>;
1153
1154 defm INT_PTX_ATOM_ADD_G_F64 : F_ATOMIC_2<Float64Regs, ".global", ".f64", ".add",
1155   atomic_load_add_f64_g, f64imm, fpimm, [hasAtomAddF64]>;
1156 defm INT_PTX_ATOM_ADD_S_F64 : F_ATOMIC_2<Float64Regs, ".shared", ".f64", ".add",
1157   atomic_load_add_f64_s, f64imm, fpimm, [hasAtomAddF64]>;
1158 defm INT_PTX_ATOM_ADD_GEN_F64 : F_ATOMIC_2<Float64Regs, "", ".f64", ".add",
1159   atomic_load_add_f64_gen, f64imm, fpimm, [hasAtomAddF64]>;
1160
1161 // atom_sub
1162
1163 def atomic_load_sub_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1164   (atomic_load_sub_32 node:$a, node:$b)>;
1165 def atomic_load_sub_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1166   (atomic_load_sub_32 node:$a, node:$b)>;
1167 def atomic_load_sub_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1168   (atomic_load_sub_32 node:$a, node:$b)>;
1169 def atomic_load_sub_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1170   (atomic_load_sub_64 node:$a, node:$b)>;
1171 def atomic_load_sub_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1172   (atomic_load_sub_64 node:$a, node:$b)>;
1173 def atomic_load_sub_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1174   (atomic_load_sub_64 node:$a, node:$b)>;
1175
1176 defm INT_PTX_ATOM_SUB_G_32 : F_ATOMIC_2_NEG<Int32Regs, ".global", "32", ".add",
1177   atomic_load_sub_32_g, i32imm>;
1178 defm INT_PTX_ATOM_SUB_G_64 : F_ATOMIC_2_NEG<Int64Regs, ".global", "64", ".add",
1179   atomic_load_sub_64_g, i64imm>;
1180 defm INT_PTX_ATOM_SUB_GEN_32 : F_ATOMIC_2_NEG<Int32Regs, "", "32", ".add",
1181   atomic_load_sub_32_gen, i32imm>;
1182 defm INT_PTX_ATOM_SUB_GEN_32_USE_G : F_ATOMIC_2_NEG<Int32Regs, ".global", "32",
1183   ".add", atomic_load_sub_32_gen, i32imm>;
1184 defm INT_PTX_ATOM_SUB_S_32 : F_ATOMIC_2_NEG<Int32Regs, ".shared", "32", ".add",
1185   atomic_load_sub_32_s, i32imm>;
1186 defm INT_PTX_ATOM_SUB_S_64 : F_ATOMIC_2_NEG<Int64Regs, ".shared", "64", ".add",
1187   atomic_load_sub_64_s, i64imm>;
1188 defm INT_PTX_ATOM_SUB_GEN_64 : F_ATOMIC_2_NEG<Int64Regs, "", "64", ".add",
1189   atomic_load_sub_64_gen, i64imm>;
1190 defm INT_PTX_ATOM_SUB_GEN_64_USE_G : F_ATOMIC_2_NEG<Int64Regs, ".global", "64",
1191   ".add", atomic_load_sub_64_gen, i64imm>;
1192
1193 // atom_swap
1194
1195 def atomic_swap_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1196   (atomic_swap_32 node:$a, node:$b)>;
1197 def atomic_swap_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1198   (atomic_swap_32 node:$a, node:$b)>;
1199 def atomic_swap_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1200   (atomic_swap_32 node:$a, node:$b)>;
1201 def atomic_swap_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1202   (atomic_swap_64 node:$a, node:$b)>;
1203 def atomic_swap_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1204   (atomic_swap_64 node:$a, node:$b)>;
1205 def atomic_swap_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1206   (atomic_swap_64 node:$a, node:$b)>;
1207
1208 defm INT_PTX_ATOM_SWAP_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".b32", ".exch",
1209   atomic_swap_32_g, i32imm, imm>;
1210 defm INT_PTX_ATOM_SWAP_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".b32", ".exch",
1211   atomic_swap_32_s, i32imm, imm>;
1212 defm INT_PTX_ATOM_SWAP_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".b32", ".exch",
1213   atomic_swap_32_gen, i32imm, imm>;
1214 defm INT_PTX_ATOM_SWAP_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".b32",
1215   ".exch", atomic_swap_32_gen, i32imm, imm>;
1216 defm INT_PTX_ATOM_SWAP_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".b64", ".exch",
1217   atomic_swap_64_g, i64imm, imm>;
1218 defm INT_PTX_ATOM_SWAP_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".b64", ".exch",
1219   atomic_swap_64_s, i64imm, imm>;
1220 defm INT_PTX_ATOM_SWAP_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".b64", ".exch",
1221   atomic_swap_64_gen, i64imm, imm>;
1222 defm INT_PTX_ATOM_SWAP_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global", ".b64",
1223   ".exch", atomic_swap_64_gen, i64imm, imm>;
1224
1225 // atom_max
1226
1227 def atomic_load_max_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b)
1228   , (atomic_load_max_32 node:$a, node:$b)>;
1229 def atomic_load_max_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1230   (atomic_load_max_32 node:$a, node:$b)>;
1231 def atomic_load_max_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1232   (atomic_load_max_32 node:$a, node:$b)>;
1233 def atomic_load_max_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b)
1234   , (atomic_load_max_64 node:$a, node:$b)>;
1235 def atomic_load_max_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1236   (atomic_load_max_64 node:$a, node:$b)>;
1237 def atomic_load_max_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1238   (atomic_load_max_64 node:$a, node:$b)>;
1239 def atomic_load_umax_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1240   (atomic_load_umax_32 node:$a, node:$b)>;
1241 def atomic_load_umax_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1242   (atomic_load_umax_32 node:$a, node:$b)>;
1243 def atomic_load_umax_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1244   (atomic_load_umax_32 node:$a, node:$b)>;
1245 def atomic_load_umax_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1246   (atomic_load_umax_64 node:$a, node:$b)>;
1247 def atomic_load_umax_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1248   (atomic_load_umax_64 node:$a, node:$b)>;
1249 def atomic_load_umax_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1250   (atomic_load_umax_64 node:$a, node:$b)>;
1251
1252 defm INT_PTX_ATOM_LOAD_MAX_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".s32",
1253   ".max", atomic_load_max_32_g, i32imm, imm>;
1254 defm INT_PTX_ATOM_LOAD_MAX_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".s32",
1255   ".max", atomic_load_max_32_s, i32imm, imm>;
1256 defm INT_PTX_ATOM_LOAD_MAX_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".s32", ".max",
1257   atomic_load_max_32_gen, i32imm, imm>;
1258 defm INT_PTX_ATOM_LOAD_MAX_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global",
1259   ".s32", ".max", atomic_load_max_32_gen, i32imm, imm>;
1260 defm INT_PTX_ATOM_LOAD_MAX_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".s64",
1261   ".max", atomic_load_max_64_g, i64imm, imm>;
1262 defm INT_PTX_ATOM_LOAD_MAX_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".s64",
1263   ".max", atomic_load_max_64_s, i64imm, imm>;
1264 defm INT_PTX_ATOM_LOAD_MAX_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".s64", ".max",
1265   atomic_load_max_64_gen, i64imm, imm>;
1266 defm INT_PTX_ATOM_LOAD_MAX_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global",
1267   ".s64", ".max", atomic_load_max_64_gen, i64imm, imm>;
1268 defm INT_PTX_ATOM_LOAD_UMAX_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".u32",
1269   ".max", atomic_load_umax_32_g, i32imm, imm>;
1270 defm INT_PTX_ATOM_LOAD_UMAX_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".u32",
1271   ".max", atomic_load_umax_32_s, i32imm, imm>;
1272 defm INT_PTX_ATOM_LOAD_UMAX_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".u32", ".max",
1273   atomic_load_umax_32_gen, i32imm, imm>;
1274 defm INT_PTX_ATOM_LOAD_UMAX_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global",
1275   ".u32", ".max", atomic_load_umax_32_gen, i32imm, imm>;
1276 defm INT_PTX_ATOM_LOAD_UMAX_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".u64",
1277   ".max", atomic_load_umax_64_g, i64imm, imm>;
1278 defm INT_PTX_ATOM_LOAD_UMAX_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".u64",
1279   ".max", atomic_load_umax_64_s, i64imm, imm>;
1280 defm INT_PTX_ATOM_LOAD_UMAX_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".u64", ".max",
1281   atomic_load_umax_64_gen, i64imm, imm>;
1282 defm INT_PTX_ATOM_LOAD_UMAX_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global",
1283   ".u64", ".max", atomic_load_umax_64_gen, i64imm, imm>;
1284
1285 // atom_min
1286
1287 def atomic_load_min_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1288   (atomic_load_min_32 node:$a, node:$b)>;
1289 def atomic_load_min_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1290   (atomic_load_min_32 node:$a, node:$b)>;
1291 def atomic_load_min_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1292   (atomic_load_min_32 node:$a, node:$b)>;
1293 def atomic_load_min_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1294   (atomic_load_min_64 node:$a, node:$b)>;
1295 def atomic_load_min_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1296   (atomic_load_min_64 node:$a, node:$b)>;
1297 def atomic_load_min_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1298   (atomic_load_min_64 node:$a, node:$b)>;
1299 def atomic_load_umin_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1300   (atomic_load_umin_32 node:$a, node:$b)>;
1301 def atomic_load_umin_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1302   (atomic_load_umin_32 node:$a, node:$b)>;
1303 def atomic_load_umin_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1304   (atomic_load_umin_32 node:$a, node:$b)>;
1305 def atomic_load_umin_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1306   (atomic_load_umin_64 node:$a, node:$b)>;
1307 def atomic_load_umin_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1308   (atomic_load_umin_64 node:$a, node:$b)>;
1309 def atomic_load_umin_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1310   (atomic_load_umin_64 node:$a, node:$b)>;
1311
1312 defm INT_PTX_ATOM_LOAD_MIN_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".s32",
1313   ".min", atomic_load_min_32_g, i32imm, imm>;
1314 defm INT_PTX_ATOM_LOAD_MIN_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".s32",
1315   ".min", atomic_load_min_32_s, i32imm, imm>;
1316 defm INT_PTX_ATOM_LOAD_MIN_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".s32", ".min",
1317   atomic_load_min_32_gen, i32imm, imm>;
1318 defm INT_PTX_ATOM_LOAD_MIN_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global",
1319   ".s32", ".min", atomic_load_min_32_gen, i32imm, imm>;
1320 defm INT_PTX_ATOM_LOAD_MIN_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".s64",
1321   ".min", atomic_load_min_64_g, i64imm, imm>;
1322 defm INT_PTX_ATOM_LOAD_MIN_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".s64",
1323   ".min", atomic_load_min_64_s, i64imm, imm>;
1324 defm INT_PTX_ATOM_LOAD_MIN_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".s64", ".min",
1325   atomic_load_min_64_gen, i64imm, imm>;
1326 defm INT_PTX_ATOM_LOAD_MIN_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global",
1327   ".s64", ".min", atomic_load_min_64_gen, i64imm, imm>;
1328 defm INT_PTX_ATOM_LOAD_UMIN_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".u32",
1329   ".min", atomic_load_umin_32_g, i32imm, imm>;
1330 defm INT_PTX_ATOM_LOAD_UMIN_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".u32",
1331   ".min", atomic_load_umin_32_s, i32imm, imm>;
1332 defm INT_PTX_ATOM_LOAD_UMIN_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".u32", ".min",
1333   atomic_load_umin_32_gen, i32imm, imm>;
1334 defm INT_PTX_ATOM_LOAD_UMIN_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global",
1335   ".u32", ".min", atomic_load_umin_32_gen, i32imm, imm>;
1336 defm INT_PTX_ATOM_LOAD_UMIN_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".u64",
1337   ".min", atomic_load_umin_64_g, i64imm, imm>;
1338 defm INT_PTX_ATOM_LOAD_UMIN_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".u64",
1339   ".min", atomic_load_umin_64_s, i64imm, imm>;
1340 defm INT_PTX_ATOM_LOAD_UMIN_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".u64", ".min",
1341   atomic_load_umin_64_gen, i64imm, imm>;
1342 defm INT_PTX_ATOM_LOAD_UMIN_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global",
1343   ".u64", ".min", atomic_load_umin_64_gen, i64imm, imm>;
1344
1345 // atom_inc  atom_dec
1346
1347 def atomic_load_inc_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1348   (int_nvvm_atomic_load_inc_32 node:$a, node:$b)>;
1349 def atomic_load_inc_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1350   (int_nvvm_atomic_load_inc_32 node:$a, node:$b)>;
1351 def atomic_load_inc_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1352   (int_nvvm_atomic_load_inc_32 node:$a, node:$b)>;
1353 def atomic_load_dec_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1354   (int_nvvm_atomic_load_dec_32 node:$a, node:$b)>;
1355 def atomic_load_dec_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1356   (int_nvvm_atomic_load_dec_32 node:$a, node:$b)>;
1357 def atomic_load_dec_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1358   (int_nvvm_atomic_load_dec_32 node:$a, node:$b)>;
1359
1360 defm INT_PTX_ATOM_INC_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".u32", ".inc",
1361   atomic_load_inc_32_g, i32imm, imm>;
1362 defm INT_PTX_ATOM_INC_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".u32", ".inc",
1363   atomic_load_inc_32_s, i32imm, imm>;
1364 defm INT_PTX_ATOM_INC_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".u32", ".inc",
1365   atomic_load_inc_32_gen, i32imm, imm>;
1366 defm INT_PTX_ATOM_INC_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".u32",
1367   ".inc", atomic_load_inc_32_gen, i32imm, imm>;
1368 defm INT_PTX_ATOM_DEC_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".u32", ".dec",
1369   atomic_load_dec_32_g, i32imm, imm>;
1370 defm INT_PTX_ATOM_DEC_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".u32", ".dec",
1371   atomic_load_dec_32_s, i32imm, imm>;
1372 defm INT_PTX_ATOM_DEC_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".u32", ".dec",
1373   atomic_load_dec_32_gen, i32imm, imm>;
1374 defm INT_PTX_ATOM_DEC_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".u32",
1375   ".dec", atomic_load_dec_32_gen, i32imm, imm>;
1376
1377 // atom_and
1378
1379 def atomic_load_and_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1380   (atomic_load_and_32 node:$a, node:$b)>;
1381 def atomic_load_and_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1382   (atomic_load_and_32 node:$a, node:$b)>;
1383 def atomic_load_and_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1384   (atomic_load_and_32 node:$a, node:$b)>;
1385 def atomic_load_and_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1386   (atomic_load_and_64 node:$a, node:$b)>;
1387 def atomic_load_and_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1388   (atomic_load_and_64 node:$a, node:$b)>;
1389 def atomic_load_and_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1390   (atomic_load_and_64 node:$a, node:$b)>;
1391
1392 defm INT_PTX_ATOM_AND_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".b32", ".and",
1393   atomic_load_and_32_g, i32imm, imm>;
1394 defm INT_PTX_ATOM_AND_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".b32", ".and",
1395   atomic_load_and_32_s, i32imm, imm>;
1396 defm INT_PTX_ATOM_AND_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".b32", ".and",
1397   atomic_load_and_32_gen, i32imm, imm>;
1398 defm INT_PTX_ATOM_AND_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".b32",
1399   ".and", atomic_load_and_32_gen, i32imm, imm>;
1400 defm INT_PTX_ATOM_AND_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".b64", ".and",
1401   atomic_load_and_64_g, i64imm, imm>;
1402 defm INT_PTX_ATOM_AND_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".b64", ".and",
1403   atomic_load_and_64_s, i64imm, imm>;
1404 defm INT_PTX_ATOM_AND_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".b64", ".and",
1405   atomic_load_and_64_gen, i64imm, imm>;
1406 defm INT_PTX_ATOM_AND_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global", ".b64",
1407   ".and", atomic_load_and_64_gen, i64imm, imm>;
1408
1409 // atom_or
1410
1411 def atomic_load_or_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1412   (atomic_load_or_32 node:$a, node:$b)>;
1413 def atomic_load_or_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1414   (atomic_load_or_32 node:$a, node:$b)>;
1415 def atomic_load_or_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1416   (atomic_load_or_32 node:$a, node:$b)>;
1417 def atomic_load_or_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1418   (atomic_load_or_64 node:$a, node:$b)>;
1419 def atomic_load_or_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1420   (atomic_load_or_64 node:$a, node:$b)>;
1421 def atomic_load_or_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1422   (atomic_load_or_64 node:$a, node:$b)>;
1423
1424 defm INT_PTX_ATOM_OR_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".b32", ".or",
1425   atomic_load_or_32_g, i32imm, imm>;
1426 defm INT_PTX_ATOM_OR_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".b32", ".or",
1427   atomic_load_or_32_gen, i32imm, imm>;
1428 defm INT_PTX_ATOM_OR_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".b32",
1429   ".or", atomic_load_or_32_gen, i32imm, imm>;
1430 defm INT_PTX_ATOM_OR_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".b32", ".or",
1431   atomic_load_or_32_s, i32imm, imm>;
1432 defm INT_PTX_ATOM_OR_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".b64", ".or",
1433   atomic_load_or_64_g, i64imm, imm>;
1434 defm INT_PTX_ATOM_OR_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".b64", ".or",
1435   atomic_load_or_64_gen, i64imm, imm>;
1436 defm INT_PTX_ATOM_OR_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global", ".b64",
1437   ".or", atomic_load_or_64_gen, i64imm, imm>;
1438 defm INT_PTX_ATOM_OR_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".b64", ".or",
1439   atomic_load_or_64_s, i64imm, imm>;
1440
1441 // atom_xor
1442
1443 def atomic_load_xor_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1444   (atomic_load_xor_32 node:$a, node:$b)>;
1445 def atomic_load_xor_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1446   (atomic_load_xor_32 node:$a, node:$b)>;
1447 def atomic_load_xor_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1448   (atomic_load_xor_32 node:$a, node:$b)>;
1449 def atomic_load_xor_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b),
1450   (atomic_load_xor_64 node:$a, node:$b)>;
1451 def atomic_load_xor_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b),
1452   (atomic_load_xor_64 node:$a, node:$b)>;
1453 def atomic_load_xor_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b),
1454   (atomic_load_xor_64 node:$a, node:$b)>;
1455
1456 defm INT_PTX_ATOM_XOR_G_32 : F_ATOMIC_2<Int32Regs, ".global", ".b32", ".xor",
1457   atomic_load_xor_32_g, i32imm, imm>;
1458 defm INT_PTX_ATOM_XOR_S_32 : F_ATOMIC_2<Int32Regs, ".shared", ".b32", ".xor",
1459   atomic_load_xor_32_s, i32imm, imm>;
1460 defm INT_PTX_ATOM_XOR_GEN_32 : F_ATOMIC_2<Int32Regs, "", ".b32", ".xor",
1461   atomic_load_xor_32_gen, i32imm, imm>;
1462 defm INT_PTX_ATOM_XOR_GEN_32_USE_G : F_ATOMIC_2<Int32Regs, ".global", ".b32",
1463   ".xor", atomic_load_xor_32_gen, i32imm, imm>;
1464 defm INT_PTX_ATOM_XOR_G_64 : F_ATOMIC_2<Int64Regs, ".global", ".b64", ".xor",
1465   atomic_load_xor_64_g, i64imm, imm>;
1466 defm INT_PTX_ATOM_XOR_S_64 : F_ATOMIC_2<Int64Regs, ".shared", ".b64", ".xor",
1467   atomic_load_xor_64_s, i64imm, imm>;
1468 defm INT_PTX_ATOM_XOR_GEN_64 : F_ATOMIC_2<Int64Regs, "", ".b64", ".xor",
1469   atomic_load_xor_64_gen, i64imm, imm>;
1470 defm INT_PTX_ATOM_XOR_GEN_64_USE_G : F_ATOMIC_2<Int64Regs, ".global", ".b64",
1471   ".xor", atomic_load_xor_64_gen, i64imm, imm>;
1472
1473 // atom_cas
1474
1475 def atomic_cmp_swap_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b, node:$c),
1476   (atomic_cmp_swap_32 node:$a, node:$b, node:$c)>;
1477 def atomic_cmp_swap_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b, node:$c),
1478   (atomic_cmp_swap_32 node:$a, node:$b, node:$c)>;
1479 def atomic_cmp_swap_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b, node:$c),
1480   (atomic_cmp_swap_32 node:$a, node:$b, node:$c)>;
1481 def atomic_cmp_swap_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b, node:$c),
1482   (atomic_cmp_swap_64 node:$a, node:$b, node:$c)>;
1483 def atomic_cmp_swap_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b, node:$c),
1484   (atomic_cmp_swap_64 node:$a, node:$b, node:$c)>;
1485 def atomic_cmp_swap_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b, node:$c),
1486   (atomic_cmp_swap_64 node:$a, node:$b, node:$c)>;
1487
1488 defm INT_PTX_ATOM_CAS_G_32 : F_ATOMIC_3<Int32Regs, ".global", ".b32", ".cas",
1489   atomic_cmp_swap_32_g, i32imm>;
1490 defm INT_PTX_ATOM_CAS_S_32 : F_ATOMIC_3<Int32Regs, ".shared", ".b32", ".cas",
1491   atomic_cmp_swap_32_s, i32imm>;
1492 defm INT_PTX_ATOM_CAS_GEN_32 : F_ATOMIC_3<Int32Regs, "", ".b32", ".cas",
1493   atomic_cmp_swap_32_gen, i32imm>;
1494 defm INT_PTX_ATOM_CAS_GEN_32_USE_G : F_ATOMIC_3<Int32Regs, ".global", ".b32",
1495   ".cas", atomic_cmp_swap_32_gen, i32imm>;
1496 defm INT_PTX_ATOM_CAS_G_64 : F_ATOMIC_3<Int64Regs, ".global", ".b64", ".cas",
1497   atomic_cmp_swap_64_g, i64imm>;
1498 defm INT_PTX_ATOM_CAS_S_64 : F_ATOMIC_3<Int64Regs, ".shared", ".b64", ".cas",
1499   atomic_cmp_swap_64_s, i64imm>;
1500 defm INT_PTX_ATOM_CAS_GEN_64 : F_ATOMIC_3<Int64Regs, "", ".b64", ".cas",
1501   atomic_cmp_swap_64_gen, i64imm>;
1502 defm INT_PTX_ATOM_CAS_GEN_64_USE_G : F_ATOMIC_3<Int64Regs, ".global", ".b64",
1503   ".cas", atomic_cmp_swap_64_gen, i64imm>;
1504
1505 // Support for scoped atomic operations.  Matches
1506 // int_nvvm_atomic_{op}_{space}_{type}_{scope}
1507 // and converts it into the appropriate instruction.
1508 // NOTE: not all possible combinations are implemented
1509 //  'space' is limited to generic as it's the only one needed to support CUDA.
1510 //  'scope' = 'gpu' is default and is handled by regular atomic instructions.
1511 class ATOM23_impl<string AsmStr, NVPTXRegClass regclass, list<Predicate> Preds,
1512                   dag ins, dag Operands>
1513       : NVPTXInst<(outs regclass:$result), ins,
1514                   AsmStr,
1515                   [(set regclass:$result, Operands)]>,
1516         Requires<Preds>;
1517
1518 // Define instruction variants for all addressing modes.
1519 multiclass ATOM2P_impl<string AsmStr,  Intrinsic Intr,
1520                        NVPTXRegClass regclass, Operand ImmType,
1521                        SDNode Imm, ValueType ImmTy,
1522                        list<Predicate> Preds> {
1523   let AddedComplexity = 1 in {
1524     def : ATOM23_impl<AsmStr, regclass, Preds,
1525                       (ins Int32Regs:$src, regclass:$b),
1526                       (Intr Int32Regs:$src, regclass:$b)>;
1527     def : ATOM23_impl<AsmStr, regclass, Preds,
1528                       (ins Int64Regs:$src, regclass:$b),
1529                       (Intr Int64Regs:$src, regclass:$b)>;
1530   }
1531   // tablegen can't infer argument types from Intrinsic (though it can
1532   // from Instruction) so we have to enforce specific type on
1533   // immediates via explicit cast to ImmTy.
1534   def : ATOM23_impl<AsmStr, regclass, Preds,
1535                     (ins Int32Regs:$src, ImmType:$b),
1536                     (Intr Int32Regs:$src, (ImmTy Imm:$b))>;
1537   def : ATOM23_impl<AsmStr, regclass, Preds,
1538                     (ins Int64Regs:$src, ImmType:$b),
1539                     (Intr Int64Regs:$src, (ImmTy Imm:$b))>;
1540 }
1541
1542 multiclass ATOM3P_impl<string AsmStr,  Intrinsic Intr,
1543                        NVPTXRegClass regclass, Operand ImmType,
1544                        SDNode Imm, ValueType ImmTy,
1545                        list<Predicate> Preds> {
1546   // Variants for register/immediate permutations of $b and $c
1547   let AddedComplexity = 2 in {
1548     def : ATOM23_impl<AsmStr, regclass, Preds,
1549                       (ins Int32Regs:$src, regclass:$b, regclass:$c),
1550                       (Intr Int32Regs:$src, regclass:$b, regclass:$c)>;
1551     def : ATOM23_impl<AsmStr, regclass, Preds,
1552                       (ins Int64Regs:$src, regclass:$b, regclass:$c),
1553                       (Intr Int64Regs:$src, regclass:$b, regclass:$c)>;
1554   }
1555   let AddedComplexity = 1 in {
1556     def : ATOM23_impl<AsmStr, regclass, Preds,
1557                       (ins Int32Regs:$src, ImmType:$b, regclass:$c),
1558                       (Intr Int32Regs:$src, (ImmTy Imm:$b), regclass:$c)>;
1559     def : ATOM23_impl<AsmStr, regclass, Preds,
1560                       (ins Int64Regs:$src, ImmType:$b, regclass:$c),
1561                       (Intr Int64Regs:$src, (ImmTy Imm:$b), regclass:$c)>;
1562     def : ATOM23_impl<AsmStr, regclass, Preds,
1563                       (ins Int32Regs:$src, regclass:$b, ImmType:$c),
1564                       (Intr Int32Regs:$src, regclass:$b, (ImmTy Imm:$c))>;
1565     def : ATOM23_impl<AsmStr, regclass, Preds,
1566                       (ins Int64Regs:$src, regclass:$b, ImmType:$c),
1567                       (Intr Int64Regs:$src, regclass:$b, (ImmTy Imm:$c))>;
1568   }
1569   def : ATOM23_impl<AsmStr, regclass, Preds,
1570                     (ins Int32Regs:$src, ImmType:$b, ImmType:$c),
1571                     (Intr Int32Regs:$src, (ImmTy Imm:$b), (ImmTy Imm:$c))>;
1572   def : ATOM23_impl<AsmStr, regclass, Preds,
1573                     (ins Int64Regs:$src, ImmType:$b, ImmType:$c),
1574                     (Intr Int64Regs:$src, (ImmTy Imm:$b), (ImmTy Imm:$c))>;
1575 }
1576
1577 // Constructs instrinsic name and instruction asm strings.
1578 multiclass ATOM2N_impl<string OpStr, string IntTypeStr, string TypeStr,
1579                        string ScopeStr, string SpaceStr,
1580                        NVPTXRegClass regclass, Operand ImmType, SDNode Imm,
1581                        ValueType ImmTy, list<Predicate> Preds> {
1582   defm : ATOM2P_impl<"atom" # !if(!eq(SpaceStr, "gen"), "", "." # SpaceStr)
1583                             # !if(!eq(ScopeStr, "gpu"), "", "." # ScopeStr)
1584                             # "." # OpStr # "." # TypeStr
1585                             # " \t$result, [$src], $b;",
1586                      !cast<Intrinsic>(
1587                             "int_nvvm_atomic_" # OpStr
1588                             # "_" # SpaceStr # "_" # IntTypeStr
1589                             # !if(!eq(ScopeStr,""), "", "_" # ScopeStr)),
1590                      regclass, ImmType, Imm, ImmTy, Preds>;
1591 }
1592 multiclass ATOM3N_impl<string OpStr, string IntTypeStr, string TypeStr,
1593                        string ScopeStr, string SpaceStr,
1594                        NVPTXRegClass regclass, Operand ImmType, SDNode Imm,
1595                        ValueType ImmTy, list<Predicate> Preds> {
1596   defm : ATOM3P_impl<"atom" # !if(!eq(SpaceStr, "gen"), "", "." # SpaceStr)
1597                             # !if(!eq(ScopeStr, "gpu"), "", "." # ScopeStr)
1598                             # "." # OpStr # "." # TypeStr
1599                             # " \t$result, [$src], $b, $c;",
1600                      !cast<Intrinsic>(
1601                             "int_nvvm_atomic_" # OpStr
1602                             # "_" # SpaceStr # "_" # IntTypeStr
1603                             # !if(!eq(ScopeStr,""), "", "_" # ScopeStr)),
1604                      regclass, ImmType, Imm, ImmTy, Preds>;
1605 }
1606
1607 // Constructs variants for different address spaces.
1608 // For now we only need variants for generic space pointers.
1609 multiclass ATOM2A_impl<string OpStr, string IntTypeStr, string TypeStr,
1610                        string ScopeStr, NVPTXRegClass regclass, Operand ImmType,
1611                        SDNode Imm, ValueType ImmTy, list<Predicate> Preds> {
1612    defm _gen_ : ATOM2N_impl<OpStr, IntTypeStr, TypeStr, ScopeStr, "gen",
1613                             regclass, ImmType, Imm, ImmTy, Preds>;
1614 }
1615 multiclass ATOM3A_impl<string OpStr, string IntTypeStr, string TypeStr,
1616                        string ScopeStr, NVPTXRegClass regclass, Operand ImmType,
1617                        SDNode Imm, ValueType ImmTy, list<Predicate> Preds> {
1618    defm _gen_ : ATOM3N_impl<OpStr, IntTypeStr, TypeStr, ScopeStr, "gen",
1619                             regclass, ImmType, Imm, ImmTy, Preds>;
1620 }
1621
1622 // Constructs variants for different scopes of atomic op.
1623 multiclass ATOM2S_impl<string OpStr, string IntTypeStr, string TypeStr,
1624                        NVPTXRegClass regclass, Operand ImmType, SDNode Imm,
1625                        ValueType ImmTy, list<Predicate> Preds> {
1626    // .gpu scope is default and is currently covered by existing
1627    // atomics w/o explicitly specified scope.
1628    defm _cta : ATOM2A_impl<OpStr, IntTypeStr, TypeStr, "cta",
1629                            regclass, ImmType, Imm, ImmTy,
1630                            !listconcat(Preds,[hasAtomScope])>;
1631    defm _sys : ATOM2A_impl<OpStr, IntTypeStr, TypeStr, "sys",
1632                            regclass, ImmType, Imm, ImmTy,
1633                            !listconcat(Preds,[hasAtomScope])>;
1634 }
1635 multiclass ATOM3S_impl<string OpStr, string IntTypeStr, string TypeStr,
1636            NVPTXRegClass regclass, Operand ImmType, SDNode Imm, ValueType ImmTy,
1637            list<Predicate> Preds> {
1638    // No need to define ".gpu"-scoped atomics.  They do the same thing
1639    // as the regular, non-scoped atomics defined elsewhere.
1640    defm _cta : ATOM3A_impl<OpStr, IntTypeStr, TypeStr, "cta",
1641                            regclass, ImmType, Imm, ImmTy,
1642                            !listconcat(Preds,[hasAtomScope])>;
1643    defm _sys : ATOM3A_impl<OpStr, IntTypeStr, TypeStr, "sys",
1644                            regclass, ImmType, Imm, ImmTy,
1645                            !listconcat(Preds,[hasAtomScope])>;
1646 }
1647
1648 // atom.add
1649 multiclass ATOM2_add_impl<string OpStr> {
1650    defm _s32  : ATOM2S_impl<OpStr, "i", "s32", Int32Regs, i32imm, imm, i32, []>;
1651    defm _u32  : ATOM2S_impl<OpStr, "i", "u32", Int32Regs, i32imm, imm, i32, []>;
1652    defm _u64  : ATOM2S_impl<OpStr, "i", "u64", Int64Regs, i64imm, imm, i64, []>;
1653    defm _f32  : ATOM2S_impl<OpStr, "f", "f32", Float32Regs, f32imm, fpimm, f32,
1654                             []>;
1655    defm _f64  : ATOM2S_impl<OpStr, "f", "f64", Float64Regs, f64imm, fpimm, f64,
1656                             [hasAtomAddF64]>;
1657 }
1658
1659 // atom.{and,or,xor}
1660 multiclass ATOM2_bitwise_impl<string OpStr> {
1661    defm _b32  : ATOM2S_impl<OpStr, "i", "b32", Int32Regs, i32imm, imm, i32, []>;
1662    defm _b64  : ATOM2S_impl<OpStr, "i", "b64", Int64Regs, i64imm, imm, i64,
1663                             [hasAtomBitwise64]>;
1664 }
1665
1666 // atom.exch
1667 multiclass ATOM2_exch_impl<string OpStr> {
1668    defm _b32 : ATOM2S_impl<OpStr, "i", "b32", Int32Regs, i32imm, imm, i32, []>;
1669    defm _b64 : ATOM2S_impl<OpStr, "i", "b64", Int64Regs, i64imm, imm, i64, []>;
1670 }
1671
1672 // atom.{min,max}
1673 multiclass ATOM2_minmax_impl<string OpStr> {
1674    defm _s32  : ATOM2S_impl<OpStr, "i", "s32", Int32Regs, i32imm, imm, i32, []>;
1675    defm _u32  : ATOM2S_impl<OpStr, "i", "u32", Int32Regs, i32imm, imm, i32, []>;
1676    defm _s64  : ATOM2S_impl<OpStr, "i", "s64", Int64Regs, i64imm, imm, i64,
1677                             [hasAtomMinMax64]>;
1678    defm _u64  : ATOM2S_impl<OpStr, "i", "u64", Int64Regs, i64imm, imm, i64,
1679                             [hasAtomMinMax64]>;
1680 }
1681
1682 // atom.{inc,dec}
1683 multiclass ATOM2_incdec_impl<string OpStr> {
1684    defm _u32  : ATOM2S_impl<OpStr, "i", "u32", Int32Regs, i32imm, imm, i32, []>;
1685 }
1686
1687 // atom.cas
1688 multiclass ATOM3_cas_impl<string OpStr> {
1689    defm _b32  : ATOM3S_impl<OpStr, "i", "b32", Int32Regs, i32imm, imm, i32, []>;
1690    defm _b64  : ATOM3S_impl<OpStr, "i", "b64", Int64Regs, i64imm, imm, i64, []>;
1691 }
1692
1693 defm INT_PTX_SATOM_ADD : ATOM2_add_impl<"add">;
1694 defm INT_PTX_SATOM_AND : ATOM2_bitwise_impl<"and">;
1695 defm INT_PTX_SATOM_CAS : ATOM3_cas_impl<"cas">;
1696 defm INT_PTX_SATOM_DEC : ATOM2_incdec_impl<"dec">;
1697 defm INT_PTX_SATOM_EXCH: ATOM2_exch_impl<"exch">;
1698 defm INT_PTX_SATOM_INC : ATOM2_incdec_impl<"inc">;
1699 defm INT_PTX_SATOM_MAX : ATOM2_minmax_impl<"max">;
1700 defm INT_PTX_SATOM_MIN : ATOM2_minmax_impl<"min">;
1701 defm INT_PTX_SATOM_OR  : ATOM2_bitwise_impl<"or">;
1702 defm INT_PTX_SATOM_XOR : ATOM2_bitwise_impl<"xor">;
1703
1704 //-----------------------------------
1705 // Support for ldu on sm_20 or later
1706 //-----------------------------------
1707
1708 // Don't annotate ldu instructions as mayLoad, as they load from memory that is
1709 // read-only in a kernel.
1710
1711 // Scalar
1712
1713 multiclass LDU_G<string TyStr, NVPTXRegClass regclass> {
1714   def areg: NVPTXInst<(outs regclass:$result), (ins Int32Regs:$src),
1715                !strconcat("ldu.global.", TyStr),
1716                       []>, Requires<[hasLDU]>;
1717   def areg64: NVPTXInst<(outs regclass:$result), (ins Int64Regs:$src),
1718                !strconcat("ldu.global.", TyStr),
1719                         []>, Requires<[hasLDU]>;
1720  def avar:  NVPTXInst<(outs regclass:$result), (ins imemAny:$src),
1721                !strconcat("ldu.global.", TyStr),
1722                       []>, Requires<[hasLDU]>;
1723  def ari :  NVPTXInst<(outs regclass:$result), (ins MEMri:$src),
1724                !strconcat("ldu.global.", TyStr),
1725                       []>, Requires<[hasLDU]>;
1726  def ari64 :  NVPTXInst<(outs regclass:$result), (ins MEMri64:$src),
1727                !strconcat("ldu.global.", TyStr),
1728                         []>, Requires<[hasLDU]>;
1729 }
1730
1731 defm INT_PTX_LDU_GLOBAL_i8  : LDU_G<"u8 \t$result, [$src];", Int16Regs>;
1732 defm INT_PTX_LDU_GLOBAL_i16 : LDU_G<"u16 \t$result, [$src];", Int16Regs>;
1733 defm INT_PTX_LDU_GLOBAL_i32 : LDU_G<"u32 \t$result, [$src];", Int32Regs>;
1734 defm INT_PTX_LDU_GLOBAL_i64 : LDU_G<"u64 \t$result, [$src];", Int64Regs>;
1735 defm INT_PTX_LDU_GLOBAL_f16 : LDU_G<"b16 \t$result, [$src];", Float16Regs>;
1736 defm INT_PTX_LDU_GLOBAL_f16x2 : LDU_G<"b32 \t$result, [$src];", Float16x2Regs>;
1737 defm INT_PTX_LDU_GLOBAL_f32 : LDU_G<"f32 \t$result, [$src];", Float32Regs>;
1738 defm INT_PTX_LDU_GLOBAL_f64 : LDU_G<"f64 \t$result, [$src];", Float64Regs>;
1739 defm INT_PTX_LDU_GLOBAL_p32 : LDU_G<"u32 \t$result, [$src];", Int32Regs>;
1740 defm INT_PTX_LDU_GLOBAL_p64 : LDU_G<"u64 \t$result, [$src];", Int64Regs>;
1741
1742 // vector
1743
1744 // Elementized vector ldu
1745 multiclass VLDU_G_ELE_V2<string TyStr, NVPTXRegClass regclass> {
1746  def _areg32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1747                      (ins Int32Regs:$src),
1748                      !strconcat("ldu.global.", TyStr), []>;
1749  def _areg64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1750                      (ins Int64Regs:$src),
1751                      !strconcat("ldu.global.", TyStr), []>;
1752  def _ari32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1753                      (ins MEMri:$src),
1754                      !strconcat("ldu.global.", TyStr), []>;
1755  def _ari64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1756                      (ins MEMri64:$src),
1757                      !strconcat("ldu.global.", TyStr), []>;
1758  def _avar: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1759                      (ins imemAny:$src),
1760                      !strconcat("ldu.global.", TyStr), []>;
1761 }
1762
1763 multiclass VLDU_G_ELE_V4<string TyStr, NVPTXRegClass regclass> {
1764  def _areg32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1765                             regclass:$dst4), (ins Int32Regs:$src),
1766                !strconcat("ldu.global.", TyStr), []>;
1767  def _areg64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1768                             regclass:$dst4), (ins Int64Regs:$src),
1769                !strconcat("ldu.global.", TyStr), []>;
1770  def _ari32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1771                             regclass:$dst4), (ins MEMri:$src),
1772                !strconcat("ldu.global.", TyStr), []>;
1773  def _ari64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1774                             regclass:$dst4), (ins MEMri64:$src),
1775                !strconcat("ldu.global.", TyStr), []>;
1776  def _avar: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1777                             regclass:$dst4), (ins imemAny:$src),
1778                !strconcat("ldu.global.", TyStr), []>;
1779 }
1780
1781 defm INT_PTX_LDU_G_v2i8_ELE
1782   : VLDU_G_ELE_V2<"v2.u8 \t{{$dst1, $dst2}}, [$src];",  Int16Regs>;
1783 defm INT_PTX_LDU_G_v2i16_ELE
1784   : VLDU_G_ELE_V2<"v2.u16 \t{{$dst1, $dst2}}, [$src];", Int16Regs>;
1785 defm INT_PTX_LDU_G_v2i32_ELE
1786   : VLDU_G_ELE_V2<"v2.u32 \t{{$dst1, $dst2}}, [$src];", Int32Regs>;
1787 defm INT_PTX_LDU_G_v2f16_ELE
1788   : VLDU_G_ELE_V2<"v2.b16 \t{{$dst1, $dst2}}, [$src];", Float16Regs>;
1789 defm INT_PTX_LDU_G_v2f16x2_ELE
1790   : VLDU_G_ELE_V2<"v2.b32 \t{{$dst1, $dst2}}, [$src];", Float16x2Regs>;
1791 defm INT_PTX_LDU_G_v2f32_ELE
1792   : VLDU_G_ELE_V2<"v2.f32 \t{{$dst1, $dst2}}, [$src];", Float32Regs>;
1793 defm INT_PTX_LDU_G_v2i64_ELE
1794   : VLDU_G_ELE_V2<"v2.u64 \t{{$dst1, $dst2}}, [$src];", Int64Regs>;
1795 defm INT_PTX_LDU_G_v2f64_ELE
1796   : VLDU_G_ELE_V2<"v2.f64 \t{{$dst1, $dst2}}, [$src];", Float64Regs>;
1797 defm INT_PTX_LDU_G_v4i8_ELE
1798   : VLDU_G_ELE_V4<"v4.u8 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", Int16Regs>;
1799 defm INT_PTX_LDU_G_v4i16_ELE
1800   : VLDU_G_ELE_V4<"v4.u16 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];",
1801     Int16Regs>;
1802 defm INT_PTX_LDU_G_v4i32_ELE
1803   : VLDU_G_ELE_V4<"v4.u32 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];",
1804     Int32Regs>;
1805 defm INT_PTX_LDU_G_v4f16_ELE
1806   : VLDU_G_ELE_V4<"v4.b16 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];",
1807     Float16Regs>;
1808 defm INT_PTX_LDU_G_v4f16x2_ELE
1809   : VLDU_G_ELE_V4<"v4.b32 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];",
1810     Float16x2Regs>;
1811 defm INT_PTX_LDU_G_v4f32_ELE
1812   : VLDU_G_ELE_V4<"v4.f32 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];",
1813     Float32Regs>;
1814
1815
1816 //-----------------------------------
1817 // Support for ldg on sm_35 or later
1818 //-----------------------------------
1819
1820 // Don't annotate ld.global.nc as mayLoad, because these loads go through the
1821 // non-coherent texture cache, and therefore the values read must be read-only
1822 // during the lifetime of the kernel.
1823
1824 multiclass LDG_G<string TyStr, NVPTXRegClass regclass> {
1825   def areg: NVPTXInst<(outs regclass:$result), (ins Int32Regs:$src),
1826                !strconcat("ld.global.nc.", TyStr),
1827                       []>, Requires<[hasLDG]>;
1828   def areg64: NVPTXInst<(outs regclass:$result), (ins Int64Regs:$src),
1829                !strconcat("ld.global.nc.", TyStr),
1830                         []>, Requires<[hasLDG]>;
1831  def avar:  NVPTXInst<(outs regclass:$result), (ins imemAny:$src),
1832                !strconcat("ld.global.nc.", TyStr),
1833                       []>, Requires<[hasLDG]>;
1834  def ari :  NVPTXInst<(outs regclass:$result), (ins MEMri:$src),
1835                !strconcat("ld.global.nc.", TyStr),
1836                       []>, Requires<[hasLDG]>;
1837  def ari64 :  NVPTXInst<(outs regclass:$result), (ins MEMri64:$src),
1838                !strconcat("ld.global.nc.", TyStr),
1839                         []>, Requires<[hasLDG]>;
1840 }
1841
1842 defm INT_PTX_LDG_GLOBAL_i8
1843   : LDG_G<"u8 \t$result, [$src];", Int16Regs>;
1844 defm INT_PTX_LDG_GLOBAL_i16
1845   : LDG_G<"u16 \t$result, [$src];", Int16Regs>;
1846 defm INT_PTX_LDG_GLOBAL_i32
1847   : LDG_G<"u32 \t$result, [$src];", Int32Regs>;
1848 defm INT_PTX_LDG_GLOBAL_i64
1849   : LDG_G<"u64 \t$result, [$src];", Int64Regs>;
1850 defm INT_PTX_LDG_GLOBAL_f16
1851   : LDG_G<"b16 \t$result, [$src];", Float16Regs>;
1852 defm INT_PTX_LDG_GLOBAL_f16x2
1853   : LDG_G<"b32 \t$result, [$src];", Float16x2Regs>;
1854 defm INT_PTX_LDG_GLOBAL_f32
1855   : LDG_G<"f32 \t$result, [$src];", Float32Regs>;
1856 defm INT_PTX_LDG_GLOBAL_f64
1857   : LDG_G<"f64 \t$result, [$src];", Float64Regs>;
1858 defm INT_PTX_LDG_GLOBAL_p32
1859   : LDG_G<"u32 \t$result, [$src];", Int32Regs>;
1860 defm INT_PTX_LDG_GLOBAL_p64
1861   : LDG_G<"u64 \t$result, [$src];", Int64Regs>;
1862
1863 // vector
1864
1865 // Elementized vector ldg
1866 multiclass VLDG_G_ELE_V2<string TyStr, NVPTXRegClass regclass> {
1867  def _areg32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1868                      (ins Int32Regs:$src),
1869                      !strconcat("ld.global.nc.", TyStr), []>;
1870  def _areg64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1871                      (ins Int64Regs:$src),
1872                      !strconcat("ld.global.nc.", TyStr), []>;
1873  def _ari32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1874                      (ins MEMri:$src),
1875                      !strconcat("ld.global.nc.", TyStr), []>;
1876  def _ari64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1877                      (ins MEMri64:$src),
1878                      !strconcat("ld.global.nc.", TyStr), []>;
1879  def _avar: NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
1880                      (ins imemAny:$src),
1881                      !strconcat("ld.global.nc.", TyStr), []>;
1882 }
1883
1884 multiclass VLDG_G_ELE_V4<string TyStr, NVPTXRegClass regclass> {
1885   def _areg32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1886                               regclass:$dst4), (ins Int32Regs:$src),
1887                !strconcat("ld.global.nc.", TyStr), []>;
1888   def _areg64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1889                                regclass:$dst4), (ins Int64Regs:$src),
1890                !strconcat("ld.global.nc.", TyStr), []>;
1891   def _ari32: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1892                               regclass:$dst4), (ins MEMri:$src),
1893                !strconcat("ld.global.nc.", TyStr), []>;
1894   def _ari64: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1895                               regclass:$dst4), (ins MEMri64:$src),
1896                !strconcat("ld.global.nc.", TyStr), []>;
1897   def _avar: NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
1898                              regclass:$dst4), (ins imemAny:$src),
1899                !strconcat("ld.global.nc.", TyStr), []>;
1900 }
1901
1902 // FIXME: 8-bit LDG should be fixed once LDG/LDU nodes are made into proper loads.
1903 defm INT_PTX_LDG_G_v2i8_ELE
1904   : VLDG_G_ELE_V2<"v2.u8 \t{{$dst1, $dst2}}, [$src];",  Int16Regs>;
1905 defm INT_PTX_LDG_G_v2i16_ELE
1906   : VLDG_G_ELE_V2<"v2.u16 \t{{$dst1, $dst2}}, [$src];", Int16Regs>;
1907 defm INT_PTX_LDG_G_v2i32_ELE
1908   : VLDG_G_ELE_V2<"v2.u32 \t{{$dst1, $dst2}}, [$src];", Int32Regs>;
1909 defm INT_PTX_LDG_G_v2f16_ELE
1910   : VLDG_G_ELE_V2<"v2.b16 \t{{$dst1, $dst2}}, [$src];", Float16Regs>;
1911 defm INT_PTX_LDG_G_v2f16x2_ELE
1912   : VLDG_G_ELE_V2<"v2.b32 \t{{$dst1, $dst2}}, [$src];", Float16x2Regs>;
1913 defm INT_PTX_LDG_G_v2f32_ELE
1914   : VLDG_G_ELE_V2<"v2.f32 \t{{$dst1, $dst2}}, [$src];", Float32Regs>;
1915 defm INT_PTX_LDG_G_v2i64_ELE
1916   : VLDG_G_ELE_V2<"v2.u64 \t{{$dst1, $dst2}}, [$src];", Int64Regs>;
1917 defm INT_PTX_LDG_G_v2f64_ELE
1918   : VLDG_G_ELE_V2<"v2.f64 \t{{$dst1, $dst2}}, [$src];", Float64Regs>;
1919 defm INT_PTX_LDG_G_v4i8_ELE
1920   : VLDG_G_ELE_V4<"v4.u8 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", Int16Regs>;
1921 defm INT_PTX_LDG_G_v4i16_ELE
1922   : VLDG_G_ELE_V4<"v4.u16 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", Int16Regs>;
1923 defm INT_PTX_LDG_G_v4i32_ELE
1924   : VLDG_G_ELE_V4<"v4.u32 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", Int32Regs>;
1925 defm INT_PTX_LDG_G_v4f16_ELE
1926   : VLDG_G_ELE_V4<"v4.b16 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", Float16Regs>;
1927 defm INT_PTX_LDG_G_v4f16x2_ELE
1928   : VLDG_G_ELE_V4<"v4.b32 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", Float16x2Regs>;
1929 defm INT_PTX_LDG_G_v4f32_ELE
1930   : VLDG_G_ELE_V4<"v4.f32 \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", Float32Regs>;
1931
1932
1933 multiclass NG_TO_G<string Str, Intrinsic Intrin> {
1934    def _yes : NVPTXInst<(outs Int32Regs:$result), (ins Int32Regs:$src),
1935           !strconcat("cvta.", Str, ".u32 \t$result, $src;"),
1936       [(set Int32Regs:$result, (Intrin Int32Regs:$src))]>;
1937    def _yes_64 : NVPTXInst<(outs Int64Regs:$result), (ins Int64Regs:$src),
1938           !strconcat("cvta.", Str, ".u64 \t$result, $src;"),
1939       [(set Int64Regs:$result, (Intrin Int64Regs:$src))]>;
1940    def _yes_6432 : NVPTXInst<(outs Int64Regs:$result), (ins Int32Regs:$src),
1941           "{{ .reg .b64 %tmp;\n\t"
1942           #"  cvt.u64.u32 \t%tmp, $src;\n\t"
1943           #"  cvta." # Str # ".u64 \t$result, %tmp; }}",
1944       [(set Int64Regs:$result, (Intrin Int32Regs:$src))]>,
1945       Requires<[useShortPtr]>;
1946 }
1947
1948 multiclass G_TO_NG<string Str, Intrinsic Intrin> {
1949    def _yes : NVPTXInst<(outs Int32Regs:$result), (ins Int32Regs:$src),
1950           !strconcat("cvta.to.", Str, ".u32 \t$result, $src;"),
1951       [(set Int32Regs:$result, (Intrin Int32Regs:$src))]>;
1952    def _yes_64 : NVPTXInst<(outs Int64Regs:$result), (ins Int64Regs:$src),
1953           !strconcat("cvta.to.", Str, ".u64 \t$result, $src;"),
1954       [(set Int64Regs:$result, (Intrin Int64Regs:$src))]>;
1955    def _yes_3264 : NVPTXInst<(outs Int32Regs:$result), (ins Int64Regs:$src),
1956           "{{ .reg .b64 %tmp;\n\t"
1957           #"  cvta.to." # Str # ".u64 \t%tmp, $src;\n\t"
1958           #"  cvt.u32.u64 \t$result, %tmp; }}",
1959       [(set Int32Regs:$result, (Intrin Int64Regs:$src))]>,
1960       Requires<[useShortPtr]>;
1961 }
1962
1963 defm cvta_local  : NG_TO_G<"local", int_nvvm_ptr_local_to_gen>;
1964 defm cvta_shared : NG_TO_G<"shared", int_nvvm_ptr_shared_to_gen>;
1965 defm cvta_global : NG_TO_G<"global", int_nvvm_ptr_global_to_gen>;
1966 defm cvta_const  : NG_TO_G<"const", int_nvvm_ptr_constant_to_gen>;
1967
1968 defm cvta_to_local   : G_TO_NG<"local", int_nvvm_ptr_gen_to_local>;
1969 defm cvta_to_shared : G_TO_NG<"shared", int_nvvm_ptr_gen_to_shared>;
1970 defm cvta_to_global : G_TO_NG<"global", int_nvvm_ptr_gen_to_global>;
1971 defm cvta_to_const  : G_TO_NG<"const", int_nvvm_ptr_gen_to_constant>;
1972
1973
1974 // nvvm.ptr.gen.to.param
1975 def nvvm_ptr_gen_to_param : NVPTXInst<(outs Int32Regs:$result),
1976   (ins Int32Regs:$src),
1977                         "mov.u32 \t$result, $src;",
1978                               [(set Int32Regs:$result,
1979                                 (int_nvvm_ptr_gen_to_param Int32Regs:$src))]>;
1980 def nvvm_ptr_gen_to_param_64 : NVPTXInst<(outs Int64Regs:$result),
1981   (ins Int64Regs:$src),
1982                         "mov.u64 \t$result, $src;",
1983                               [(set Int64Regs:$result,
1984                                 (int_nvvm_ptr_gen_to_param Int64Regs:$src))]>;
1985
1986
1987 // nvvm.move intrinsicc
1988 def nvvm_move_i16 : NVPTXInst<(outs Int16Regs:$r), (ins Int16Regs:$s),
1989                              "mov.b16 \t$r, $s;",
1990                              [(set Int16Regs:$r,
1991                                (int_nvvm_move_i16 Int16Regs:$s))]>;
1992 def nvvm_move_i32 : NVPTXInst<(outs Int32Regs:$r), (ins Int32Regs:$s),
1993                              "mov.b32 \t$r, $s;",
1994                              [(set Int32Regs:$r,
1995                                (int_nvvm_move_i32 Int32Regs:$s))]>;
1996 def nvvm_move_i64 : NVPTXInst<(outs Int64Regs:$r), (ins Int64Regs:$s),
1997                              "mov.b64 \t$r, $s;",
1998                              [(set Int64Regs:$r,
1999                                (int_nvvm_move_i64 Int64Regs:$s))]>;
2000 def nvvm_move_float : NVPTXInst<(outs Float32Regs:$r), (ins Float32Regs:$s),
2001                              "mov.f32 \t$r, $s;",
2002                              [(set Float32Regs:$r,
2003                                (int_nvvm_move_float Float32Regs:$s))]>;
2004 def nvvm_move_double : NVPTXInst<(outs Float64Regs:$r), (ins Float64Regs:$s),
2005                              "mov.f64 \t$r, $s;",
2006                              [(set Float64Regs:$r,
2007                                (int_nvvm_move_double Float64Regs:$s))]>;
2008 def nvvm_move_ptr32 : NVPTXInst<(outs Int32Regs:$r), (ins Int32Regs:$s),
2009                              "mov.u32 \t$r, $s;",
2010                              [(set Int32Regs:$r,
2011                                (int_nvvm_move_ptr Int32Regs:$s))]>;
2012 def nvvm_move_ptr64 : NVPTXInst<(outs Int64Regs:$r), (ins Int64Regs:$s),
2013                              "mov.u64 \t$r, $s;",
2014                              [(set Int64Regs:$r,
2015                                (int_nvvm_move_ptr Int64Regs:$s))]>;
2016
2017 // @TODO: Are these actually needed, or will we always just see symbols
2018 // copied to registers first?
2019 /*def nvvm_move_sym32 : NVPTXInst<(outs Int32Regs:$r), (ins imem:$s),
2020                              "mov.u32 \t$r, $s;",
2021                              [(set Int32Regs:$r,
2022                              (int_nvvm_move_ptr texternalsym:$s))]>;
2023 def nvvm_move_sym64 : NVPTXInst<(outs Int64Regs:$r), (ins imem:$s),
2024                              "mov.u64 \t$r, $s;",
2025                              [(set Int64Regs:$r,
2026                              (int_nvvm_move_ptr texternalsym:$s))]>;*/
2027
2028
2029 // MoveParam        %r1, param
2030 // ptr_local_to_gen %r2, %r1
2031 // ptr_gen_to_local %r3, %r2
2032 // ->
2033 // mov %r1, param
2034
2035 // @TODO: Revisit this.  There is a type
2036 // contradiction between iPTRAny and iPTR for the addr defs, so the move_sym
2037 // instructions are not currently defined. However, we can use the ptr
2038 // variants and the asm printer will do the right thing.
2039 def : Pat<(i64 (int_nvvm_ptr_gen_to_local (int_nvvm_ptr_local_to_gen
2040                 (MoveParam texternalsym:$src)))),
2041                (nvvm_move_ptr64  texternalsym:$src)>;
2042 def : Pat<(i32 (int_nvvm_ptr_gen_to_local (int_nvvm_ptr_local_to_gen
2043                 (MoveParam texternalsym:$src)))),
2044                (nvvm_move_ptr32  texternalsym:$src)>;
2045
2046 def texsurf_handles
2047   : NVPTXInst<(outs Int64Regs:$result), (ins imem:$src),
2048               "mov.u64 \t$result, $src;", []>;
2049
2050 //-----------------------------------
2051 // Compiler Error Warn
2052 // - Just ignore them in codegen
2053 //-----------------------------------
2054
2055 def INT_NVVM_COMPILER_WARN_32 : NVPTXInst<(outs), (ins Int32Regs:$a),
2056                 "// llvm.nvvm.compiler.warn()",
2057                 [(int_nvvm_compiler_warn Int32Regs:$a)]>;
2058 def INT_NVVM_COMPILER_WARN_64 : NVPTXInst<(outs), (ins Int64Regs:$a),
2059                 "// llvm.nvvm.compiler.warn()",
2060                 [(int_nvvm_compiler_warn Int64Regs:$a)]>;
2061 def INT_NVVM_COMPILER_ERROR_32 : NVPTXInst<(outs), (ins Int32Regs:$a),
2062                 "// llvm.nvvm.compiler.error()",
2063                 [(int_nvvm_compiler_error Int32Regs:$a)]>;
2064 def INT_NVVM_COMPILER_ERROR_64 : NVPTXInst<(outs), (ins Int64Regs:$a),
2065                 "// llvm.nvvm.compiler.error()",
2066                 [(int_nvvm_compiler_error Int64Regs:$a)]>;
2067
2068
2069 // isspacep
2070
2071 def ISSPACEP_CONST_32
2072   : NVPTXInst<(outs Int1Regs:$d), (ins Int32Regs:$a),
2073               "isspacep.const \t$d, $a;",
2074               [(set Int1Regs:$d, (int_nvvm_isspacep_const Int32Regs:$a))]>,
2075     Requires<[hasPTX31]>;
2076 def ISSPACEP_CONST_64
2077   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
2078               "isspacep.const \t$d, $a;",
2079               [(set Int1Regs:$d, (int_nvvm_isspacep_const Int64Regs:$a))]>,
2080     Requires<[hasPTX31]>;
2081 def ISSPACEP_GLOBAL_32
2082   : NVPTXInst<(outs Int1Regs:$d), (ins Int32Regs:$a),
2083               "isspacep.global \t$d, $a;",
2084               [(set Int1Regs:$d, (int_nvvm_isspacep_global Int32Regs:$a))]>;
2085 def ISSPACEP_GLOBAL_64
2086   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
2087               "isspacep.global \t$d, $a;",
2088               [(set Int1Regs:$d, (int_nvvm_isspacep_global Int64Regs:$a))]>;
2089 def ISSPACEP_LOCAL_32
2090   : NVPTXInst<(outs Int1Regs:$d), (ins Int32Regs:$a),
2091               "isspacep.local \t$d, $a;",
2092               [(set Int1Regs:$d, (int_nvvm_isspacep_local Int32Regs:$a))]>;
2093 def ISSPACEP_LOCAL_64
2094   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
2095               "isspacep.local \t$d, $a;",
2096               [(set Int1Regs:$d, (int_nvvm_isspacep_local Int64Regs:$a))]>;
2097 def ISSPACEP_SHARED_32
2098   : NVPTXInst<(outs Int1Regs:$d), (ins Int32Regs:$a),
2099               "isspacep.shared \t$d, $a;",
2100               [(set Int1Regs:$d, (int_nvvm_isspacep_shared Int32Regs:$a))]>;
2101 def ISSPACEP_SHARED_64
2102   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
2103               "isspacep.shared \t$d, $a;",
2104               [(set Int1Regs:$d, (int_nvvm_isspacep_shared Int64Regs:$a))]>;
2105
2106
2107 // Special register reads
2108 def MOV_SPECIAL : NVPTXInst<(outs Int32Regs:$d),
2109                             (ins SpecialRegs:$r),
2110                             "mov.b32 \t$d, $r;", []>;
2111
2112 def : Pat<(int_nvvm_read_ptx_sreg_envreg0), (MOV_SPECIAL ENVREG0)>;
2113 def : Pat<(int_nvvm_read_ptx_sreg_envreg1), (MOV_SPECIAL ENVREG1)>;
2114 def : Pat<(int_nvvm_read_ptx_sreg_envreg2), (MOV_SPECIAL ENVREG2)>;
2115 def : Pat<(int_nvvm_read_ptx_sreg_envreg3), (MOV_SPECIAL ENVREG3)>;
2116 def : Pat<(int_nvvm_read_ptx_sreg_envreg4), (MOV_SPECIAL ENVREG4)>;
2117 def : Pat<(int_nvvm_read_ptx_sreg_envreg5), (MOV_SPECIAL ENVREG5)>;
2118 def : Pat<(int_nvvm_read_ptx_sreg_envreg6), (MOV_SPECIAL ENVREG6)>;
2119 def : Pat<(int_nvvm_read_ptx_sreg_envreg7), (MOV_SPECIAL ENVREG7)>;
2120 def : Pat<(int_nvvm_read_ptx_sreg_envreg8), (MOV_SPECIAL ENVREG8)>;
2121 def : Pat<(int_nvvm_read_ptx_sreg_envreg9), (MOV_SPECIAL ENVREG9)>;
2122 def : Pat<(int_nvvm_read_ptx_sreg_envreg10), (MOV_SPECIAL ENVREG10)>;
2123 def : Pat<(int_nvvm_read_ptx_sreg_envreg11), (MOV_SPECIAL ENVREG11)>;
2124 def : Pat<(int_nvvm_read_ptx_sreg_envreg12), (MOV_SPECIAL ENVREG12)>;
2125 def : Pat<(int_nvvm_read_ptx_sreg_envreg13), (MOV_SPECIAL ENVREG13)>;
2126 def : Pat<(int_nvvm_read_ptx_sreg_envreg14), (MOV_SPECIAL ENVREG14)>;
2127 def : Pat<(int_nvvm_read_ptx_sreg_envreg15), (MOV_SPECIAL ENVREG15)>;
2128 def : Pat<(int_nvvm_read_ptx_sreg_envreg16), (MOV_SPECIAL ENVREG16)>;
2129 def : Pat<(int_nvvm_read_ptx_sreg_envreg17), (MOV_SPECIAL ENVREG17)>;
2130 def : Pat<(int_nvvm_read_ptx_sreg_envreg18), (MOV_SPECIAL ENVREG18)>;
2131 def : Pat<(int_nvvm_read_ptx_sreg_envreg19), (MOV_SPECIAL ENVREG19)>;
2132 def : Pat<(int_nvvm_read_ptx_sreg_envreg20), (MOV_SPECIAL ENVREG20)>;
2133 def : Pat<(int_nvvm_read_ptx_sreg_envreg21), (MOV_SPECIAL ENVREG21)>;
2134 def : Pat<(int_nvvm_read_ptx_sreg_envreg22), (MOV_SPECIAL ENVREG22)>;
2135 def : Pat<(int_nvvm_read_ptx_sreg_envreg23), (MOV_SPECIAL ENVREG23)>;
2136 def : Pat<(int_nvvm_read_ptx_sreg_envreg24), (MOV_SPECIAL ENVREG24)>;
2137 def : Pat<(int_nvvm_read_ptx_sreg_envreg25), (MOV_SPECIAL ENVREG25)>;
2138 def : Pat<(int_nvvm_read_ptx_sreg_envreg26), (MOV_SPECIAL ENVREG26)>;
2139 def : Pat<(int_nvvm_read_ptx_sreg_envreg27), (MOV_SPECIAL ENVREG27)>;
2140 def : Pat<(int_nvvm_read_ptx_sreg_envreg28), (MOV_SPECIAL ENVREG28)>;
2141 def : Pat<(int_nvvm_read_ptx_sreg_envreg29), (MOV_SPECIAL ENVREG29)>;
2142 def : Pat<(int_nvvm_read_ptx_sreg_envreg30), (MOV_SPECIAL ENVREG30)>;
2143 def : Pat<(int_nvvm_read_ptx_sreg_envreg31), (MOV_SPECIAL ENVREG31)>;
2144
2145
2146 // rotate builtin support
2147
2148 def ROTATE_B32_HW_IMM
2149   : NVPTXInst<(outs Int32Regs:$dst),
2150               (ins  Int32Regs:$src, i32imm:$amt),
2151               "shf.l.wrap.b32 \t$dst, $src, $src, $amt;",
2152               [(set Int32Regs:$dst,
2153                  (int_nvvm_rotate_b32 Int32Regs:$src, (i32 imm:$amt)))]>,
2154               Requires<[hasHWROT32]> ;
2155
2156 def ROTATE_B32_HW_REG
2157   : NVPTXInst<(outs Int32Regs:$dst),
2158               (ins  Int32Regs:$src, Int32Regs:$amt),
2159               "shf.l.wrap.b32 \t$dst, $src, $src, $amt;",
2160               [(set Int32Regs:$dst,
2161                  (int_nvvm_rotate_b32 Int32Regs:$src, Int32Regs:$amt))]>,
2162               Requires<[hasHWROT32]> ;
2163
2164 def : Pat<(int_nvvm_rotate_b32 Int32Regs:$src, (i32 imm:$amt)),
2165           (ROT32imm_sw Int32Regs:$src, imm:$amt, (SUB_FRM_32 node:$amt))>,
2166       Requires<[noHWROT32]> ;
2167
2168 def : Pat<(int_nvvm_rotate_b32 Int32Regs:$src, Int32Regs:$amt),
2169           (ROTL32reg_sw Int32Regs:$src, Int32Regs:$amt)>,
2170       Requires<[noHWROT32]> ;
2171
2172 let hasSideEffects = 0 in {
2173   def GET_LO_INT64 : NVPTXInst<(outs Int32Regs:$dst), (ins Int64Regs:$src),
2174     !strconcat("{{\n\t",
2175                ".reg .b32 %dummy;\n\t",
2176                "mov.b64 \t{$dst,%dummy}, $src;\n\t",
2177                "}}"),
2178           []> ;
2179
2180   def GET_HI_INT64 : NVPTXInst<(outs Int32Regs:$dst), (ins Int64Regs:$src),
2181     !strconcat("{{\n\t",
2182                ".reg .b32 %dummy;\n\t",
2183                "mov.b64 \t{%dummy,$dst}, $src;\n\t",
2184                "}}"),
2185           []> ;
2186 }
2187
2188 let hasSideEffects = 0 in {
2189   def PACK_TWO_INT32
2190     : NVPTXInst<(outs Int64Regs:$dst), (ins Int32Regs:$lo, Int32Regs:$hi),
2191                 "mov.b64 \t$dst, {{$lo, $hi}};", []> ;
2192 }
2193
2194 def : Pat<(int_nvvm_swap_lo_hi_b64 Int64Regs:$src),
2195           (PACK_TWO_INT32 (GET_HI_INT64 Int64Regs:$src),
2196                           (GET_LO_INT64 Int64Regs:$src))> ;
2197
2198 // Funnel shift, requires >= sm_32.  Does not trap if amt is out of range, so
2199 // no side effects.
2200 let hasSideEffects = 0 in {
2201   def SHF_L_WRAP_B32_IMM
2202     : NVPTXInst<(outs Int32Regs:$dst),
2203                 (ins  Int32Regs:$lo, Int32Regs:$hi, i32imm:$amt),
2204                 "shf.l.wrap.b32 \t$dst, $lo, $hi, $amt;",[]>,
2205       Requires<[hasHWROT32]>;
2206
2207   def SHF_L_WRAP_B32_REG
2208     : NVPTXInst<(outs Int32Regs:$dst),
2209                 (ins  Int32Regs:$lo, Int32Regs:$hi, Int32Regs:$amt),
2210                 "shf.l.wrap.b32 \t$dst, $lo, $hi, $amt;",[]>,
2211       Requires<[hasHWROT32]>;
2212
2213   def SHF_R_WRAP_B32_IMM
2214     : NVPTXInst<(outs Int32Regs:$dst),
2215                 (ins  Int32Regs:$lo, Int32Regs:$hi, i32imm:$amt),
2216                 "shf.r.wrap.b32 \t$dst, $lo, $hi, $amt;",[]>,
2217       Requires<[hasHWROT32]>;
2218
2219   def SHF_R_WRAP_B32_REG
2220     : NVPTXInst<(outs Int32Regs:$dst),
2221                 (ins  Int32Regs:$lo, Int32Regs:$hi, Int32Regs:$amt),
2222                 "shf.r.wrap.b32 \t$dst, $lo, $hi, $amt;",[]>,
2223       Requires<[hasHWROT32]>;
2224 }
2225
2226 // HW version of rotate 64
2227 def : Pat<(int_nvvm_rotate_b64 Int64Regs:$src, (i32 imm:$amt)),
2228           (PACK_TWO_INT32
2229             (SHF_L_WRAP_B32_IMM (GET_HI_INT64 Int64Regs:$src),
2230                                 (GET_LO_INT64 Int64Regs:$src), imm:$amt),
2231             (SHF_L_WRAP_B32_IMM (GET_LO_INT64 Int64Regs:$src),
2232                                 (GET_HI_INT64 Int64Regs:$src), imm:$amt))>,
2233       Requires<[hasHWROT32]>;
2234
2235 def : Pat<(int_nvvm_rotate_b64 Int64Regs:$src, Int32Regs:$amt),
2236           (PACK_TWO_INT32
2237             (SHF_L_WRAP_B32_REG (GET_HI_INT64 Int64Regs:$src),
2238                                 (GET_LO_INT64 Int64Regs:$src), Int32Regs:$amt),
2239             (SHF_L_WRAP_B32_REG (GET_LO_INT64 Int64Regs:$src),
2240                                (GET_HI_INT64 Int64Regs:$src), Int32Regs:$amt))>,
2241       Requires<[hasHWROT32]>;
2242
2243
2244 def : Pat<(int_nvvm_rotate_right_b64 Int64Regs:$src, (i32 imm:$amt)),
2245           (PACK_TWO_INT32
2246             (SHF_R_WRAP_B32_IMM (GET_LO_INT64 Int64Regs:$src),
2247                                 (GET_HI_INT64 Int64Regs:$src), imm:$amt),
2248             (SHF_R_WRAP_B32_IMM (GET_HI_INT64 Int64Regs:$src),
2249                                 (GET_LO_INT64 Int64Regs:$src), imm:$amt))>,
2250       Requires<[hasHWROT32]>;
2251
2252 def : Pat<(int_nvvm_rotate_right_b64 Int64Regs:$src, Int32Regs:$amt),
2253           (PACK_TWO_INT32
2254             (SHF_R_WRAP_B32_REG (GET_LO_INT64 Int64Regs:$src),
2255                                 (GET_HI_INT64 Int64Regs:$src), Int32Regs:$amt),
2256             (SHF_R_WRAP_B32_REG (GET_HI_INT64 Int64Regs:$src),
2257                                (GET_LO_INT64 Int64Regs:$src), Int32Regs:$amt))>,
2258       Requires<[hasHWROT32]>;
2259
2260 // SW version of rotate 64
2261 def : Pat<(int_nvvm_rotate_b64 Int64Regs:$src, (i32 imm:$amt)),
2262           (ROT64imm_sw Int64Regs:$src, imm:$amt, (SUB_FRM_32 node:$amt))>,
2263       Requires<[noHWROT32]>;
2264 def : Pat<(int_nvvm_rotate_b64 Int64Regs:$src, Int32Regs:$amt),
2265           (ROTL64reg_sw Int64Regs:$src, Int32Regs:$amt)>,
2266       Requires<[noHWROT32]>;
2267 def : Pat<(int_nvvm_rotate_right_b64 Int64Regs:$src, (i32 imm:$amt)),
2268           (ROT64imm_sw Int64Regs:$src, (SUB_FRM_64 node:$amt), imm:$amt)>,
2269       Requires<[noHWROT32]>;
2270 def : Pat<(int_nvvm_rotate_right_b64 Int64Regs:$src, Int32Regs:$amt),
2271           (ROTR64reg_sw Int64Regs:$src, Int32Regs:$amt)>,
2272       Requires<[noHWROT32]>;
2273
2274
2275 //-----------------------------------
2276 // Texture Intrinsics
2277 //-----------------------------------
2278
2279 // NOTE: For Fermi support, any new texture/surface/sampler intrinsics must be
2280 // also defined in NVPTXReplaceImageHandles.cpp
2281
2282 // texmode_independent
2283 let IsTex = 1, IsTexModeUnified = 0 in {
2284 // Texture fetch instructions using handles
2285 def TEX_1D_F32_S32
2286   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2287                     Float32Regs:$b, Float32Regs:$a),
2288               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x),
2289               "tex.1d.v4.f32.s32 \t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}];",
2290               []>;
2291 def TEX_1D_F32_F32
2292   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2293                     Float32Regs:$b, Float32Regs:$a),
2294               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x),
2295               "tex.1d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}];",
2296               []>;
2297 def TEX_1D_F32_F32_LEVEL
2298   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2299                     Float32Regs:$b, Float32Regs:$a),
2300               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$lod),
2301               "tex.level.1d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2302               "[$t, $s, \\{$x\\}], $lod;",
2303               []>;
2304 def TEX_1D_F32_F32_GRAD
2305   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2306                     Float32Regs:$b, Float32Regs:$a),
2307               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x,
2308                    Float32Regs:$gradx, Float32Regs:$grady),
2309               "tex.grad.1d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2310               "[$t, $s, \\{$x\\}], \\{$gradx\\}, \\{$grady\\};",
2311               []>;
2312 def TEX_1D_S32_S32
2313   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2314                     Int32Regs:$b, Int32Regs:$a),
2315               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x),
2316               "tex.1d.v4.s32.s32 \t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}];",
2317               []>;
2318 def TEX_1D_S32_F32
2319   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2320                     Int32Regs:$b, Int32Regs:$a),
2321               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x),
2322               "tex.1d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}];",
2323               []>;
2324 def TEX_1D_S32_F32_LEVEL
2325   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2326                     Int32Regs:$b, Int32Regs:$a),
2327               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x,
2328                    Float32Regs:$lod),
2329               "tex.level.1d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2330               "[$t, $s, \\{$x\\}], $lod;",
2331               []>;
2332 def TEX_1D_S32_F32_GRAD
2333   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2334                     Int32Regs:$b, Int32Regs:$a),
2335               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x,
2336                    Float32Regs:$gradx, Float32Regs:$grady),
2337               "tex.grad.1d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2338               "[$t, $s, \\{$x\\}], \\{$gradx\\}, \\{$grady\\};",
2339               []>;
2340 def TEX_1D_U32_S32
2341   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2342                     Int32Regs:$b, Int32Regs:$a),
2343               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x),
2344               "tex.1d.v4.u32.s32 \t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}];",
2345               []>;
2346 def TEX_1D_U32_F32
2347   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2348                     Int32Regs:$b, Int32Regs:$a),
2349               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x),
2350               "tex.1d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, [$t, $s, \\{$x\\}];",
2351               []>;
2352 def TEX_1D_U32_F32_LEVEL
2353   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2354                     Int32Regs:$b, Int32Regs:$a),
2355               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x,
2356                    Float32Regs:$lod),
2357               "tex.level.1d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2358               "[$t, $s, \\{$x\\}], $lod;",
2359               []>;
2360 def TEX_1D_U32_F32_GRAD
2361   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2362                     Int32Regs:$b, Int32Regs:$a),
2363               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x,
2364                    Float32Regs:$gradx, Float32Regs:$grady),
2365               "tex.grad.1d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2366               "[$t, $s, \\{$x\\}], \\{$gradx\\}, \\{$grady\\};",
2367               []>;
2368
2369 def TEX_1D_ARRAY_F32_S32
2370   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2371                     Float32Regs:$b, Float32Regs:$a),
2372               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
2373               "tex.a1d.v4.f32.s32 \t\\{$r, $g, $b, $a\\}, "
2374               "[$t, $s, \\{$l, $x\\}];",
2375               []>;
2376 def TEX_1D_ARRAY_F32_F32
2377   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2378                     Float32Regs:$b, Float32Regs:$a),
2379               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x),
2380               "tex.a1d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2381               "[$t, $s, \\{$l, $x\\}];",
2382               []>;
2383 def TEX_1D_ARRAY_F32_F32_LEVEL
2384   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2385                     Float32Regs:$b, Float32Regs:$a),
2386               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2387                    Float32Regs:$lod),
2388               "tex.level.a1d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2389               "[$t, $s, \\{$l, $x\\}], $lod;",
2390               []>;
2391 def TEX_1D_ARRAY_F32_F32_GRAD
2392   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2393                     Float32Regs:$b, Float32Regs:$a),
2394               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2395                    Float32Regs:$gradx, Float32Regs:$grady),
2396               "tex.grad.a1d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2397               "[$t, $s, \\{$l, $x\\}], \\{$gradx\\}, \\{$grady\\};",
2398               []>;
2399 def TEX_1D_ARRAY_S32_S32
2400   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2401                     Int32Regs:$b, Int32Regs:$a),
2402               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
2403               "tex.a1d.v4.s32.s32 \t\\{$r, $g, $b, $a\\}, "
2404               "[$t, $s, \\{$l, $x\\}];",
2405               []>;
2406 def TEX_1D_ARRAY_S32_F32
2407   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2408                     Int32Regs:$b, Int32Regs:$a),
2409               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x),
2410               "tex.a1d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2411               "[$t, $s, \\{$l, $x\\}];",
2412               []>;
2413 def TEX_1D_ARRAY_S32_F32_LEVEL
2414   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2415                     Int32Regs:$b, Int32Regs:$a),
2416               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2417                    Float32Regs:$lod),
2418               "tex.level.a1d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2419               "[$t, $s, \\{$l, $x\\}], $lod;",
2420               []>;
2421 def TEX_1D_ARRAY_S32_F32_GRAD
2422   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2423                     Int32Regs:$b, Int32Regs:$a),
2424               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2425                    Float32Regs:$gradx, Float32Regs:$grady),
2426               "tex.grad.a1d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2427               "[$t, $s, \\{$l, $x\\}], \\{$gradx\\}, \\{$grady\\};",
2428               []>;
2429 def TEX_1D_ARRAY_U32_S32
2430   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2431                     Int32Regs:$b, Int32Regs:$a),
2432               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
2433               "tex.a1d.v4.u32.s32 \t\\{$r, $g, $b, $a\\}, "
2434               "[$t, $s, \\{$l, $x\\}];",
2435               []>;
2436 def TEX_1D_ARRAY_U32_F32
2437   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2438                     Int32Regs:$b, Int32Regs:$a),
2439               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x),
2440               "tex.a1d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2441               "[$t, $s, \\{$l, $x\\}];",
2442               []>;
2443 def TEX_1D_ARRAY_U32_F32_LEVEL
2444   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2445                     Int32Regs:$b, Int32Regs:$a),
2446               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2447                    Float32Regs:$lod),
2448               "tex.level.a1d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2449               "[$t, $s, \\{$l, $x\\}], $lod;",
2450               []>;
2451 def TEX_1D_ARRAY_U32_F32_GRAD
2452   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2453                     Int32Regs:$b, Int32Regs:$a),
2454               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2455                    Float32Regs:$gradx, Float32Regs:$grady),
2456               "tex.grad.a1d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2457               "[$t, $s, \\{$l, $x\\}], \\{$gradx\\}, \\{$grady\\};",
2458               []>;
2459
2460 def TEX_2D_F32_S32
2461   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2462                     Float32Regs:$b, Float32Regs:$a),
2463               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
2464               "tex.2d.v4.f32.s32 \t\\{$r, $g, $b, $a\\}, "
2465               "[$t, $s, \\{$x, $y\\}];",
2466               []>;
2467 def TEX_2D_F32_F32
2468   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2469                     Float32Regs:$b, Float32Regs:$a),
2470               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2471               "tex.2d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2472               "[$t, $s, \\{$x, $y\\}];",
2473               []>;
2474 def TEX_2D_F32_F32_LEVEL
2475   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2476                     Float32Regs:$b, Float32Regs:$a),
2477               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2478                    Float32Regs:$lod),
2479               "tex.level.2d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2480               "[$t, $s, \\{$x, $y\\}], $lod;",
2481               []>;
2482 def TEX_2D_F32_F32_GRAD
2483   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2484                     Float32Regs:$b, Float32Regs:$a),
2485               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2486                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2487                    Float32Regs:$grady0, Float32Regs:$grady1),
2488               "tex.grad.2d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2489               "[$t, $s, \\{$x, $y\\}], \\{$gradx0, $gradx1\\}, "
2490               "\\{$grady0, $grady1\\};",
2491               []>;
2492 def TEX_2D_S32_S32
2493   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2494                     Int32Regs:$b, Int32Regs:$a),
2495               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
2496               "tex.2d.v4.s32.s32 \t\\{$r, $g, $b, $a\\}, "
2497               "[$t, $s, \\{$x, $y\\}];",
2498               []>;
2499 def TEX_2D_S32_F32
2500   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2501                     Int32Regs:$b, Int32Regs:$a),
2502               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2503               "tex.2d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2504               "[$t, $s, \\{$x, $y\\}];",
2505               []>;
2506 def TEX_2D_S32_F32_LEVEL
2507   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2508                     Int32Regs:$b, Int32Regs:$a),
2509               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2510                    Float32Regs:$lod),
2511               "tex.level.2d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2512               "[$t, $s, \\{$x, $y\\}], $lod;",
2513               []>;
2514 def TEX_2D_S32_F32_GRAD
2515   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2516                     Int32Regs:$b, Int32Regs:$a),
2517               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2518                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2519                    Float32Regs:$grady0, Float32Regs:$grady1),
2520               "tex.grad.2d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2521               "[$t, $s, \\{$x, $y\\}], \\{$gradx0, $gradx1\\}, "
2522               "\\{$grady0, $grady1\\};",
2523               []>;
2524 def TEX_2D_U32_S32
2525   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2526                     Int32Regs:$b, Int32Regs:$a),
2527               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
2528               "tex.2d.v4.u32.s32 \t\\{$r, $g, $b, $a\\}, "
2529               "[$t, $s, \\{$x, $y\\}];",
2530               []>;
2531 def TEX_2D_U32_F32
2532   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2533                     Int32Regs:$b, Int32Regs:$a),
2534               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2535               "tex.2d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2536               "[$t, $s, \\{$x, $y\\}];",
2537               []>;
2538 def TEX_2D_U32_F32_LEVEL
2539   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2540                     Int32Regs:$b, Int32Regs:$a),
2541               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2542                    Float32Regs:$lod),
2543               "tex.level.2d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2544               "[$t, $s, \\{$x, $y\\}], $lod;",
2545               []>;
2546 def TEX_2D_U32_F32_GRAD
2547   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2548                     Int32Regs:$b, Int32Regs:$a),
2549               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2550                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2551                    Float32Regs:$grady0, Float32Regs:$grady1),
2552               "tex.grad.2d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2553               "[$t, $s, \\{$x, $y\\}], \\{$gradx0, $gradx1\\}, "
2554               "\\{$grady0, $grady1\\};",
2555               []>;
2556
2557 def TEX_2D_ARRAY_F32_S32
2558   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2559                     Float32Regs:$b, Float32Regs:$a),
2560               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
2561                    Int32Regs:$y),
2562               "tex.a2d.v4.f32.s32 \t\\{$r, $g, $b, $a\\}, "
2563               "[$t, $s, \\{$l, $x, $y, $y\\}];",
2564               []>;
2565 def TEX_2D_ARRAY_F32_F32
2566   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2567                     Float32Regs:$b, Float32Regs:$a),
2568               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2569                    Float32Regs:$y),
2570               "tex.a2d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2571               "[$t, $s, \\{$l, $x, $y, $y\\}];",
2572               []>;
2573 def TEX_2D_ARRAY_F32_F32_LEVEL
2574   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2575                     Float32Regs:$b, Float32Regs:$a),
2576               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2577                    Float32Regs:$y, Float32Regs:$lod),
2578               "tex.level.a2d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2579               "[$t, $s, \\{$l, $x, $y, $y\\}], $lod;",
2580               []>;
2581 def TEX_2D_ARRAY_F32_F32_GRAD
2582   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2583                     Float32Regs:$b, Float32Regs:$a),
2584               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2585                    Float32Regs:$y, Float32Regs:$gradx0, Float32Regs:$gradx1,
2586                    Float32Regs:$grady0, Float32Regs:$grady1),
2587               "tex.grad.a2d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2588               "[$t, $s, \\{$l, $x, $y, $y\\}], \\{$gradx0, $gradx1\\}, "
2589               "\\{$grady0, $grady1\\};",
2590               []>;
2591 def TEX_2D_ARRAY_S32_S32
2592   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2593                     Int32Regs:$b, Int32Regs:$a),
2594               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
2595                    Int32Regs:$y),
2596               "tex.a2d.v4.s32.s32 \t\\{$r, $g, $b, $a\\}, "
2597               "[$t, $s, \\{$l, $x, $y, $y\\}];",
2598               []>;
2599 def TEX_2D_ARRAY_S32_F32
2600   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2601                     Int32Regs:$b, Int32Regs:$a),
2602               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2603                    Float32Regs:$y),
2604               "tex.a2d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2605               "[$t, $s, \\{$l, $x, $y, $y\\}];",
2606               []>;
2607 def TEX_2D_ARRAY_S32_F32_LEVEL
2608   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2609                     Int32Regs:$b, Int32Regs:$a),
2610               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2611                    Float32Regs:$y, Float32Regs:$lod),
2612               "tex.level.a2d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2613               "[$t, $s, \\{$l, $x, $y, $y\\}], $lod;",
2614               []>;
2615 def TEX_2D_ARRAY_S32_F32_GRAD
2616   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2617                     Int32Regs:$b, Int32Regs:$a),
2618               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2619                    Float32Regs:$y,
2620                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2621                    Float32Regs:$grady0, Float32Regs:$grady1),
2622               "tex.grad.a2d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2623               "[$t, $s, \\{$l, $x, $y, $y\\}], \\{$gradx0, $gradx1\\}, "
2624               "\\{$grady0, $grady1\\};",
2625               []>;
2626 def TEX_2D_ARRAY_U32_S32
2627   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2628                     Int32Regs:$b, Int32Regs:$a),
2629               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
2630                    Int32Regs:$y),
2631               "tex.a2d.v4.u32.s32 \t\\{$r, $g, $b, $a\\}, "
2632               "[$t, $s, \\{$l, $x, $y, $y\\}];",
2633               []>;
2634 def TEX_2D_ARRAY_U32_F32
2635   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2636                     Int32Regs:$b, Int32Regs:$a),
2637               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2638                    Float32Regs:$y),
2639               "tex.a2d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2640               "[$t, $s, \\{$l, $x, $y, $y\\}];",
2641               []>;
2642 def TEX_2D_ARRAY_U32_F32_LEVEL
2643   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2644                     Int32Regs:$b, Int32Regs:$a),
2645               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2646                    Float32Regs:$y, Float32Regs:$lod),
2647               "tex.level.a2d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2648               "[$t, $s, \\{$l, $x, $y, $y\\}], $lod;",
2649               []>;
2650 def TEX_2D_ARRAY_U32_F32_GRAD
2651   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2652                     Int32Regs:$b, Int32Regs:$a),
2653               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l, Float32Regs:$x,
2654                    Float32Regs:$y,
2655                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2656                    Float32Regs:$grady0, Float32Regs:$grady1),
2657               "tex.grad.a2d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2658               "[$t, $s, \\{$l, $x, $y, $y\\}], \\{$gradx0, $gradx1\\}, "
2659               "\\{$grady0, $grady1\\};",
2660               []>;
2661
2662 def TEX_3D_F32_S32
2663   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2664                     Float32Regs:$b, Float32Regs:$a),
2665               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
2666                    Int32Regs:$z),
2667               "tex.3d.v4.f32.s32 \t\\{$r, $g, $b, $a\\}, "
2668               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2669               []>;
2670 def TEX_3D_F32_F32
2671   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2672                     Float32Regs:$b, Float32Regs:$a),
2673               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2674                    Float32Regs:$z),
2675               "tex.3d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2676               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2677               []>;
2678 def TEX_3D_F32_F32_LEVEL
2679   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2680                     Float32Regs:$b, Float32Regs:$a),
2681               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2682                    Float32Regs:$z, Float32Regs:$lod),
2683               "tex.level.3d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2684               "[$t, $s, \\{$x, $y, $z, $z\\}], $lod;",
2685               []>;
2686 def TEX_3D_F32_F32_GRAD
2687   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2688                     Float32Regs:$b, Float32Regs:$a),
2689               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2690                    Float32Regs:$z,
2691                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2692                    Float32Regs:$gradx2, Float32Regs:$grady0,
2693                    Float32Regs:$grady1, Float32Regs:$grady2),
2694               "tex.grad.3d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2695               "[$t, $s, \\{$x, $y, $z, $z\\}], "
2696               "\\{$gradx0, $gradx1, $gradx2, $gradx2\\}, "
2697               "\\{$grady0, $grady1, $grady2, $grady2\\};",
2698               []>;
2699 def TEX_3D_S32_S32
2700   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2701                     Int32Regs:$b, Int32Regs:$a),
2702               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
2703                    Int32Regs:$z),
2704               "tex.3d.v4.s32.s32 \t\\{$r, $g, $b, $a\\}, "
2705               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2706               []>;
2707 def TEX_3D_S32_F32
2708   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2709                     Int32Regs:$b, Int32Regs:$a),
2710               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2711                    Float32Regs:$z),
2712               "tex.3d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2713               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2714               []>;
2715 def TEX_3D_S32_F32_LEVEL
2716   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2717                     Int32Regs:$b, Int32Regs:$a),
2718               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2719                    Float32Regs:$z, Float32Regs:$lod),
2720               "tex.level.3d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2721               "[$t, $s, \\{$x, $y, $z, $z\\}], $lod;",
2722               []>;
2723 def TEX_3D_S32_F32_GRAD
2724   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2725                     Int32Regs:$b, Int32Regs:$a),
2726               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2727                    Float32Regs:$z,
2728                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2729                    Float32Regs:$gradx2, Float32Regs:$grady0,
2730                    Float32Regs:$grady1, Float32Regs:$grady2),
2731               "tex.grad.3d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2732               "[$t, $s, \\{$x, $y, $z, $z\\}], "
2733               "\\{$gradx0, $gradx1, $gradx2, $gradx2\\}, "
2734               "\\{$grady0, $grady1, $grady2, $grady2\\};",
2735               []>;
2736 def TEX_3D_U32_S32
2737   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2738                     Int32Regs:$b, Int32Regs:$a),
2739               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
2740                    Int32Regs:$z),
2741               "tex.3d.v4.u32.s32 \t\\{$r, $g, $b, $a\\}, "
2742               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2743               []>;
2744 def TEX_3D_U32_F32
2745   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2746                     Int32Regs:$b, Int32Regs:$a),
2747               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2748                    Float32Regs:$z),
2749               "tex.3d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2750               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2751               []>;
2752 def TEX_3D_U32_F32_LEVEL
2753   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2754                     Int32Regs:$b, Int32Regs:$a),
2755               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2756                    Float32Regs:$z, Float32Regs:$lod),
2757               "tex.level.3d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2758               "[$t, $s, \\{$x, $y, $z, $z\\}], $lod;",
2759               []>;
2760 def TEX_3D_U32_F32_GRAD
2761   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2762                     Int32Regs:$b, Int32Regs:$a),
2763               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y,
2764                    Float32Regs:$z,
2765                    Float32Regs:$gradx0, Float32Regs:$gradx1,
2766                    Float32Regs:$gradx2, Float32Regs:$grady0,
2767                    Float32Regs:$grady1, Float32Regs:$grady2),
2768               "tex.grad.3d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2769               "[$t, $s, \\{$x, $y, $z, $z\\}], "
2770               "\\{$gradx0, $gradx1, $gradx2, $gradx2\\}, "
2771               "\\{$grady0, $grady1, $grady2, $grady2\\};",
2772               []>;
2773
2774 def TEX_CUBE_F32_F32
2775   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2776                     Float32Regs:$b, Float32Regs:$a),
2777               (ins Int64Regs:$t, Int64Regs:$s,
2778                Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
2779               "tex.cube.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2780               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2781               []>;
2782 def TEX_CUBE_F32_F32_LEVEL
2783   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2784                     Float32Regs:$b, Float32Regs:$a),
2785               (ins Int64Regs:$t, Int64Regs:$s,
2786                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
2787                    Float32Regs:$lod),
2788               "tex.level.cube.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2789               "[$t, $s, \\{$x, $y, $z, $z\\}], $lod;",
2790               []>;
2791 def TEX_CUBE_S32_F32
2792   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2793                     Int32Regs:$b, Int32Regs:$a),
2794               (ins Int64Regs:$t, Int64Regs:$s,
2795                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
2796               "tex.cube.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2797               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2798               []>;
2799 def TEX_CUBE_S32_F32_LEVEL
2800   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2801                     Int32Regs:$b, Int32Regs:$a),
2802               (ins Int64Regs:$t, Int64Regs:$s,
2803                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
2804                    Float32Regs:$lod),
2805               "tex.level.cube.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2806               "[$t, $s, \\{$x, $y, $z, $z\\}], $lod;",
2807               []>;
2808 def TEX_CUBE_U32_F32
2809   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2810                     Int32Regs:$b, Int32Regs:$a),
2811               (ins Int64Regs:$t, Int64Regs:$s,
2812                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
2813               "tex.cube.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2814               "[$t, $s, \\{$x, $y, $z, $z\\}];",
2815               []>;
2816 def TEX_CUBE_U32_F32_LEVEL
2817   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2818                     Int32Regs:$b, Int32Regs:$a),
2819               (ins Int64Regs:$t, Int64Regs:$s,
2820                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
2821                    Float32Regs:$lod),
2822               "tex.level.cube.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2823               "[$t, $s, \\{$x, $y, $z, $z\\}], $lod;",
2824               []>;
2825
2826 def TEX_CUBE_ARRAY_F32_F32
2827   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2828                     Float32Regs:$b, Float32Regs:$a),
2829               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l,
2830                Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
2831               "tex.acube.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2832               "[$t, $s, \\{$l, $x, $y, $z\\}];",
2833               []>;
2834 def TEX_CUBE_ARRAY_F32_F32_LEVEL
2835   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2836                     Float32Regs:$b, Float32Regs:$a),
2837               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l,
2838                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
2839                    Float32Regs:$lod),
2840               "tex.level.acube.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2841               "[$t, $s, \\{$l, $x, $y, $z\\}], $lod;",
2842               []>;
2843 def TEX_CUBE_ARRAY_S32_F32
2844   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2845                     Int32Regs:$b, Int32Regs:$a),
2846               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l,
2847                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
2848               "tex.acube.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2849               "[$t, $s, \\{$l, $x, $y, $z\\}];",
2850               []>;
2851 def TEX_CUBE_ARRAY_S32_F32_LEVEL
2852   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2853                     Int32Regs:$b, Int32Regs:$a),
2854               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l,
2855                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
2856                    Float32Regs:$lod),
2857               "tex.level.acube.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
2858               "[$t, $s, \\{$l, $x, $y, $z\\}], $lod;",
2859               []>;
2860 def TEX_CUBE_ARRAY_U32_F32
2861   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2862                     Int32Regs:$b, Int32Regs:$a),
2863               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l,
2864                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
2865               "tex.acube.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2866               "[$t, $s, \\{$l, $x, $y, $z\\}];",
2867               []>;
2868 def TEX_CUBE_ARRAY_U32_F32_LEVEL
2869   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2870                     Int32Regs:$b, Int32Regs:$a),
2871               (ins Int64Regs:$t, Int64Regs:$s, Int32Regs:$l,
2872                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
2873                    Float32Regs:$lod),
2874               "tex.level.acube.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
2875               "[$t, $s, \\{$l, $x, $y, $z\\}], $lod;",
2876               []>;
2877
2878 def TLD4_R_2D_F32_F32
2879   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
2880                     Float32Regs:$v2, Float32Regs:$v3),
2881               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2882               "tld4.r.2d.v4.f32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
2883               "[$t, $s, \\{$x, $y\\}];",
2884               []>;
2885 def TLD4_G_2D_F32_F32
2886   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
2887                     Float32Regs:$v2, Float32Regs:$v3),
2888               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2889               "tld4.g.2d.v4.f32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
2890               "[$t, $s, \\{$x, $y\\}];",
2891               []>;
2892 def TLD4_B_2D_F32_F32
2893   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
2894                     Float32Regs:$v2, Float32Regs:$v3),
2895               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2896               "tld4.b.2d.v4.f32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
2897               "[$t, $s, \\{$x, $y\\}];",
2898               []>;
2899 def TLD4_A_2D_F32_F32
2900   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
2901                     Float32Regs:$v2, Float32Regs:$v3),
2902               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2903               "tld4.a.2d.v4.f32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
2904               "[$t, $s, \\{$x, $y\\}];",
2905               []>;
2906 def TLD4_R_2D_S32_F32
2907   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2908                     Int32Regs:$v2, Int32Regs:$v3),
2909               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2910               "tld4.r.2d.v4.s32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
2911               "[$t, $s, \\{$x, $y\\}];",
2912               []>;
2913 def TLD4_G_2D_S32_F32
2914   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2915                     Int32Regs:$v2, Int32Regs:$v3),
2916               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2917               "tld4.g.2d.v4.s32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
2918               "[$t, $s, \\{$x, $y\\}];",
2919               []>;
2920 def TLD4_B_2D_S32_F32
2921   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2922                     Int32Regs:$v2, Int32Regs:$v3),
2923               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2924               "tld4.b.2d.v4.s32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
2925               "[$t, $s, \\{$x, $y\\}];",
2926               []>;
2927 def TLD4_A_2D_S32_F32
2928   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2929                     Int32Regs:$v2, Int32Regs:$v3),
2930               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2931               "tld4.a.2d.v4.s32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
2932               "[$t, $s, \\{$x, $y\\}];",
2933               []>;
2934 def TLD4_R_2D_U32_F32
2935   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2936                     Int32Regs:$v2, Int32Regs:$v3),
2937               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2938               "tld4.r.2d.v4.u32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
2939               "[$t, $s, \\{$x, $y\\}];",
2940               []>;
2941 def TLD4_G_2D_U32_F32
2942   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2943                     Int32Regs:$v2, Int32Regs:$v3),
2944               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2945               "tld4.g.2d.v4.u32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
2946               "[$t, $s, \\{$x, $y\\}];",
2947               []>;
2948 def TLD4_B_2D_U32_F32
2949   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2950                     Int32Regs:$v2, Int32Regs:$v3),
2951               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2952               "tld4.b.2d.v4.u32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
2953               "[$t, $s, \\{$x, $y\\}];",
2954               []>;
2955 def TLD4_A_2D_U32_F32
2956   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
2957                     Int32Regs:$v2, Int32Regs:$v3),
2958               (ins Int64Regs:$t, Int64Regs:$s, Float32Regs:$x, Float32Regs:$y),
2959               "tld4.a.2d.v4.u32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
2960               "[$t, $s, \\{$x, $y\\}];",
2961               []>;
2962 }
2963
2964
2965 // texmode_unified
2966 let IsTex = 1, IsTexModeUnified = 1 in {
2967 // Texture fetch instructions using handles
2968 def TEX_UNIFIED_1D_F32_S32
2969   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2970                     Float32Regs:$b, Float32Regs:$a),
2971               (ins Int64Regs:$t, Int32Regs:$x),
2972               "tex.1d.v4.f32.s32 \t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}];",
2973               []>;
2974 def TEX_UNIFIED_1D_F32_F32
2975   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2976                     Float32Regs:$b, Float32Regs:$a),
2977               (ins Int64Regs:$t, Float32Regs:$x),
2978               "tex.1d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}];",
2979               []>;
2980 def TEX_UNIFIED_1D_F32_F32_LEVEL
2981   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2982                     Float32Regs:$b, Float32Regs:$a),
2983               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$lod),
2984               "tex.level.1d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2985               "[$t, \\{$x\\}], $lod;",
2986               []>;
2987 def TEX_UNIFIED_1D_F32_F32_GRAD
2988   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
2989                     Float32Regs:$b, Float32Regs:$a),
2990               (ins Int64Regs:$t, Float32Regs:$x,
2991                    Float32Regs:$gradx, Float32Regs:$grady),
2992               "tex.grad.1d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
2993               "[$t, \\{$x\\}], \\{$gradx\\}, \\{$grady\\};",
2994               []>;
2995 def TEX_UNIFIED_1D_S32_S32
2996   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
2997                     Int32Regs:$b, Int32Regs:$a),
2998               (ins Int64Regs:$t, Int32Regs:$x),
2999               "tex.1d.v4.s32.s32 \t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}];",
3000               []>;
3001 def TEX_UNIFIED_1D_S32_F32
3002   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3003                     Int32Regs:$b, Int32Regs:$a),
3004               (ins Int64Regs:$t, Float32Regs:$x),
3005               "tex.1d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}];",
3006               []>;
3007 def TEX_UNIFIED_1D_S32_F32_LEVEL
3008   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3009                     Int32Regs:$b, Int32Regs:$a),
3010               (ins Int64Regs:$t, Float32Regs:$x,
3011                    Float32Regs:$lod),
3012               "tex.level.1d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3013               "[$t, \\{$x\\}], $lod;",
3014               []>;
3015 def TEX_UNIFIED_1D_S32_F32_GRAD
3016   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3017                     Int32Regs:$b, Int32Regs:$a),
3018               (ins Int64Regs:$t, Float32Regs:$x,
3019                    Float32Regs:$gradx, Float32Regs:$grady),
3020               "tex.grad.1d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3021               "[$t, \\{$x\\}], \\{$gradx\\}, \\{$grady\\};",
3022               []>;
3023 def TEX_UNIFIED_1D_U32_S32
3024   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3025                     Int32Regs:$b, Int32Regs:$a),
3026               (ins Int64Regs:$t, Int32Regs:$x),
3027               "tex.1d.v4.u32.s32 \t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}];",
3028               []>;
3029 def TEX_UNIFIED_1D_U32_F32
3030   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3031                     Int32Regs:$b, Int32Regs:$a),
3032               (ins Int64Regs:$t, Float32Regs:$x),
3033               "tex.1d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, [$t, \\{$x\\}];",
3034               []>;
3035 def TEX_UNIFIED_1D_U32_F32_LEVEL
3036   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3037                     Int32Regs:$b, Int32Regs:$a),
3038               (ins Int64Regs:$t, Float32Regs:$x,
3039                    Float32Regs:$lod),
3040               "tex.level.1d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3041               "[$t, \\{$x\\}], $lod;",
3042               []>;
3043 def TEX_UNIFIED_1D_U32_F32_GRAD
3044   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3045                     Int32Regs:$b, Int32Regs:$a),
3046               (ins Int64Regs:$t, Float32Regs:$x,
3047                    Float32Regs:$gradx, Float32Regs:$grady),
3048               "tex.grad.1d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3049               "[$t, \\{$x\\}], \\{$gradx\\}, \\{$grady\\};",
3050               []>;
3051
3052 def TEX_UNIFIED_1D_ARRAY_F32_S32
3053   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3054                     Float32Regs:$b, Float32Regs:$a),
3055               (ins Int64Regs:$t, Int32Regs:$l, Int32Regs:$x),
3056               "tex.a1d.v4.f32.s32 \t\\{$r, $g, $b, $a\\}, "
3057               "[$t, \\{$l, $x\\}];",
3058               []>;
3059 def TEX_UNIFIED_1D_ARRAY_F32_F32
3060   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3061                     Float32Regs:$b, Float32Regs:$a),
3062               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x),
3063               "tex.a1d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3064               "[$t, \\{$l, $x\\}];",
3065               []>;
3066 def TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL
3067   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3068                     Float32Regs:$b, Float32Regs:$a),
3069               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3070                    Float32Regs:$lod),
3071               "tex.level.a1d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3072               "[$t, \\{$l, $x\\}], $lod;",
3073               []>;
3074 def TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD
3075   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3076                     Float32Regs:$b, Float32Regs:$a),
3077               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3078                    Float32Regs:$gradx, Float32Regs:$grady),
3079               "tex.grad.a1d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3080               "[$t, \\{$l, $x\\}], \\{$gradx\\}, \\{$grady\\};",
3081               []>;
3082 def TEX_UNIFIED_1D_ARRAY_S32_S32
3083   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3084                     Int32Regs:$b, Int32Regs:$a),
3085               (ins Int64Regs:$t, Int32Regs:$l, Int32Regs:$x),
3086               "tex.a1d.v4.s32.s32 \t\\{$r, $g, $b, $a\\}, "
3087               "[$t, \\{$l, $x\\}];",
3088               []>;
3089 def TEX_UNIFIED_1D_ARRAY_S32_F32
3090   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3091                     Int32Regs:$b, Int32Regs:$a),
3092               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x),
3093               "tex.a1d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3094               "[$t, \\{$l, $x\\}];",
3095               []>;
3096 def TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL
3097   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3098                     Int32Regs:$b, Int32Regs:$a),
3099               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3100                    Float32Regs:$lod),
3101               "tex.level.a1d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3102               "[$t, \\{$l, $x\\}], $lod;",
3103               []>;
3104 def TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD
3105   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3106                     Int32Regs:$b, Int32Regs:$a),
3107               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3108                    Float32Regs:$gradx, Float32Regs:$grady),
3109               "tex.grad.a1d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3110               "[$t, \\{$l, $x\\}], \\{$gradx\\}, \\{$grady\\};",
3111               []>;
3112 def TEX_UNIFIED_1D_ARRAY_U32_S32
3113   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3114                     Int32Regs:$b, Int32Regs:$a),
3115               (ins Int64Regs:$t, Int32Regs:$l, Int32Regs:$x),
3116               "tex.a1d.v4.u32.s32 \t\\{$r, $g, $b, $a\\}, "
3117               "[$t, \\{$l, $x\\}];",
3118               []>;
3119 def TEX_UNIFIED_1D_ARRAY_U32_F32
3120   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3121                     Int32Regs:$b, Int32Regs:$a),
3122               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x),
3123               "tex.a1d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3124               "[$t, \\{$l, $x\\}];",
3125               []>;
3126 def TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL
3127   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3128                     Int32Regs:$b, Int32Regs:$a),
3129               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3130                    Float32Regs:$lod),
3131               "tex.level.a1d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3132               "[$t, \\{$l, $x\\}], $lod;",
3133               []>;
3134 def TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD
3135   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3136                     Int32Regs:$b, Int32Regs:$a),
3137               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3138                    Float32Regs:$gradx, Float32Regs:$grady),
3139               "tex.grad.a1d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3140               "[$t, \\{$l, $x\\}], \\{$gradx\\}, \\{$grady\\};",
3141               []>;
3142
3143 def TEX_UNIFIED_2D_F32_S32
3144   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3145                     Float32Regs:$b, Float32Regs:$a),
3146               (ins Int64Regs:$t, Int32Regs:$x, Int32Regs:$y),
3147               "tex.2d.v4.f32.s32 \t\\{$r, $g, $b, $a\\}, "
3148               "[$t, \\{$x, $y\\}];",
3149               []>;
3150 def TEX_UNIFIED_2D_F32_F32
3151   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3152                     Float32Regs:$b, Float32Regs:$a),
3153               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3154               "tex.2d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3155               "[$t, \\{$x, $y\\}];",
3156               []>;
3157 def TEX_UNIFIED_2D_F32_F32_LEVEL
3158   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3159                     Float32Regs:$b, Float32Regs:$a),
3160               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3161                    Float32Regs:$lod),
3162               "tex.level.2d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3163               "[$t, \\{$x, $y\\}], $lod;",
3164               []>;
3165 def TEX_UNIFIED_2D_F32_F32_GRAD
3166   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3167                     Float32Regs:$b, Float32Regs:$a),
3168               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3169                    Float32Regs:$gradx0, Float32Regs:$gradx1,
3170                    Float32Regs:$grady0, Float32Regs:$grady1),
3171               "tex.grad.2d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3172               "[$t, \\{$x, $y\\}], \\{$gradx0, $gradx1\\}, "
3173               "\\{$grady0, $grady1\\};",
3174               []>;
3175 def TEX_UNIFIED_2D_S32_S32
3176   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3177                     Int32Regs:$b, Int32Regs:$a),
3178               (ins Int64Regs:$t, Int32Regs:$x, Int32Regs:$y),
3179               "tex.2d.v4.s32.s32 \t\\{$r, $g, $b, $a\\}, "
3180               "[$t, \\{$x, $y\\}];",
3181               []>;
3182 def TEX_UNIFIED_2D_S32_F32
3183   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3184                     Int32Regs:$b, Int32Regs:$a),
3185               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3186               "tex.2d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3187               "[$t, \\{$x, $y\\}];",
3188               []>;
3189 def TEX_UNIFIED_2D_S32_F32_LEVEL
3190   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3191                     Int32Regs:$b, Int32Regs:$a),
3192               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3193                    Float32Regs:$lod),
3194               "tex.level.2d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3195               "[$t, \\{$x, $y\\}], $lod;",
3196               []>;
3197 def TEX_UNIFIED_2D_S32_F32_GRAD
3198   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3199                     Int32Regs:$b, Int32Regs:$a),
3200               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3201                    Float32Regs:$gradx0, Float32Regs:$gradx1,
3202                    Float32Regs:$grady0, Float32Regs:$grady1),
3203               "tex.grad.2d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3204               "[$t, \\{$x, $y\\}], \\{$gradx0, $gradx1\\}, "
3205               "\\{$grady0, $grady1\\};",
3206               []>;
3207 def TEX_UNIFIED_2D_U32_S32
3208   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3209                     Int32Regs:$b, Int32Regs:$a),
3210               (ins Int64Regs:$t, Int32Regs:$x, Int32Regs:$y),
3211               "tex.2d.v4.u32.s32 \t\\{$r, $g, $b, $a\\}, "
3212               "[$t, \\{$x, $y\\}];",
3213               []>;
3214 def TEX_UNIFIED_2D_U32_F32
3215   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3216                     Int32Regs:$b, Int32Regs:$a),
3217               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3218               "tex.2d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3219               "[$t, \\{$x, $y\\}];",
3220               []>;
3221 def TEX_UNIFIED_2D_U32_F32_LEVEL
3222   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3223                     Int32Regs:$b, Int32Regs:$a),
3224               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3225                    Float32Regs:$lod),
3226               "tex.level.2d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3227               "[$t, \\{$x, $y\\}], $lod;",
3228               []>;
3229 def TEX_UNIFIED_2D_U32_F32_GRAD
3230   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3231                     Int32Regs:$b, Int32Regs:$a),
3232               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3233                    Float32Regs:$gradx0, Float32Regs:$gradx1,
3234                    Float32Regs:$grady0, Float32Regs:$grady1),
3235               "tex.grad.2d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3236               "[$t, \\{$x, $y\\}], \\{$gradx0, $gradx1\\}, "
3237               "\\{$grady0, $grady1\\};",
3238               []>;
3239
3240 def TEX_UNIFIED_2D_ARRAY_F32_S32
3241   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3242                     Float32Regs:$b, Float32Regs:$a),
3243               (ins Int64Regs:$t, Int32Regs:$l, Int32Regs:$x,
3244                    Int32Regs:$y),
3245               "tex.a2d.v4.f32.s32 \t\\{$r, $g, $b, $a\\}, "
3246               "[$t, \\{$l, $x, $y, $y\\}];",
3247               []>;
3248 def TEX_UNIFIED_2D_ARRAY_F32_F32
3249   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3250                     Float32Regs:$b, Float32Regs:$a),
3251               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3252                    Float32Regs:$y),
3253               "tex.a2d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3254               "[$t, \\{$l, $x, $y, $y\\}];",
3255               []>;
3256 def TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL
3257   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3258                     Float32Regs:$b, Float32Regs:$a),
3259               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3260                    Float32Regs:$y, Float32Regs:$lod),
3261               "tex.level.a2d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3262               "[$t, \\{$l, $x, $y, $y\\}], $lod;",
3263               []>;
3264 def TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD
3265   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3266                     Float32Regs:$b, Float32Regs:$a),
3267               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3268                    Float32Regs:$y, Float32Regs:$gradx0, Float32Regs:$gradx1,
3269                    Float32Regs:$grady0, Float32Regs:$grady1),
3270               "tex.grad.a2d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3271               "[$t, \\{$l, $x, $y, $y\\}], \\{$gradx0, $gradx1\\}, "
3272               "\\{$grady0, $grady1\\};",
3273               []>;
3274 def TEX_UNIFIED_2D_ARRAY_S32_S32
3275   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3276                     Int32Regs:$b, Int32Regs:$a),
3277               (ins Int64Regs:$t, Int32Regs:$l, Int32Regs:$x,
3278                    Int32Regs:$y),
3279               "tex.a2d.v4.s32.s32 \t\\{$r, $g, $b, $a\\}, "
3280               "[$t, \\{$l, $x, $y, $y\\}];",
3281               []>;
3282 def TEX_UNIFIED_2D_ARRAY_S32_F32
3283   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3284                     Int32Regs:$b, Int32Regs:$a),
3285               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3286                    Float32Regs:$y),
3287               "tex.a2d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3288               "[$t, \\{$l, $x, $y, $y\\}];",
3289               []>;
3290 def TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL
3291   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3292                     Int32Regs:$b, Int32Regs:$a),
3293               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3294                    Float32Regs:$y, Float32Regs:$lod),
3295               "tex.level.a2d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3296               "[$t, \\{$l, $x, $y, $y\\}], $lod;",
3297               []>;
3298 def TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD
3299   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3300                     Int32Regs:$b, Int32Regs:$a),
3301               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3302                    Float32Regs:$y,
3303                    Float32Regs:$gradx0, Float32Regs:$gradx1,
3304                    Float32Regs:$grady0, Float32Regs:$grady1),
3305               "tex.grad.a2d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3306               "[$t, \\{$l, $x, $y, $y\\}], \\{$gradx0, $gradx1\\}, "
3307               "\\{$grady0, $grady1\\};",
3308               []>;
3309 def TEX_UNIFIED_2D_ARRAY_U32_S32
3310   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3311                     Int32Regs:$b, Int32Regs:$a),
3312               (ins Int64Regs:$t, Int32Regs:$l, Int32Regs:$x,
3313                    Int32Regs:$y),
3314               "tex.a2d.v4.u32.s32 \t\\{$r, $g, $b, $a\\}, "
3315               "[$t, \\{$l, $x, $y, $y\\}];",
3316               []>;
3317 def TEX_UNIFIED_2D_ARRAY_U32_F32
3318   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3319                     Int32Regs:$b, Int32Regs:$a),
3320               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3321                    Float32Regs:$y),
3322               "tex.a2d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3323               "[$t, \\{$l, $x, $y, $y\\}];",
3324               []>;
3325 def TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL
3326   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3327                     Int32Regs:$b, Int32Regs:$a),
3328               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3329                    Float32Regs:$y, Float32Regs:$lod),
3330               "tex.level.a2d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3331               "[$t, \\{$l, $x, $y, $y\\}], $lod;",
3332               []>;
3333 def TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD
3334   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3335                     Int32Regs:$b, Int32Regs:$a),
3336               (ins Int64Regs:$t, Int32Regs:$l, Float32Regs:$x,
3337                    Float32Regs:$y,
3338                    Float32Regs:$gradx0, Float32Regs:$gradx1,
3339                    Float32Regs:$grady0, Float32Regs:$grady1),
3340               "tex.grad.a2d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3341               "[$t, \\{$l, $x, $y, $y\\}], \\{$gradx0, $gradx1\\}, "
3342               "\\{$grady0, $grady1\\};",
3343               []>;
3344
3345 def TEX_UNIFIED_3D_F32_S32
3346   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3347                     Float32Regs:$b, Float32Regs:$a),
3348               (ins Int64Regs:$t, Int32Regs:$x, Int32Regs:$y,
3349                    Int32Regs:$z),
3350               "tex.3d.v4.f32.s32 \t\\{$r, $g, $b, $a\\}, "
3351               "[$t, \\{$x, $y, $z, $z\\}];",
3352               []>;
3353 def TEX_UNIFIED_3D_F32_F32
3354   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3355                     Float32Regs:$b, Float32Regs:$a),
3356               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3357                    Float32Regs:$z),
3358               "tex.3d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3359               "[$t, \\{$x, $y, $z, $z\\}];",
3360               []>;
3361 def TEX_UNIFIED_3D_F32_F32_LEVEL
3362   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3363                     Float32Regs:$b, Float32Regs:$a),
3364               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3365                    Float32Regs:$z, Float32Regs:$lod),
3366               "tex.level.3d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3367               "[$t, \\{$x, $y, $z, $z\\}], $lod;",
3368               []>;
3369 def TEX_UNIFIED_3D_F32_F32_GRAD
3370   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3371                     Float32Regs:$b, Float32Regs:$a),
3372               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3373                    Float32Regs:$z,
3374                    Float32Regs:$gradx0, Float32Regs:$gradx1,
3375                    Float32Regs:$gradx2, Float32Regs:$grady0,
3376                    Float32Regs:$grady1, Float32Regs:$grady2),
3377               "tex.grad.3d.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3378               "[$t, \\{$x, $y, $z, $z\\}], "
3379               "\\{$gradx0, $gradx1, $gradx2, $gradx2\\}, "
3380               "\\{$grady0, $grady1, $grady2, $grady2\\};",
3381               []>;
3382 def TEX_UNIFIED_3D_S32_S32
3383   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3384                     Int32Regs:$b, Int32Regs:$a),
3385               (ins Int64Regs:$t, Int32Regs:$x, Int32Regs:$y,
3386                    Int32Regs:$z),
3387               "tex.3d.v4.s32.s32 \t\\{$r, $g, $b, $a\\}, "
3388               "[$t, \\{$x, $y, $z, $z\\}];",
3389               []>;
3390 def TEX_UNIFIED_3D_S32_F32
3391   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3392                     Int32Regs:$b, Int32Regs:$a),
3393               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3394                    Float32Regs:$z),
3395               "tex.3d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3396               "[$t, \\{$x, $y, $z, $z\\}];",
3397               []>;
3398 def TEX_UNIFIED_3D_S32_F32_LEVEL
3399   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3400                     Int32Regs:$b, Int32Regs:$a),
3401               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3402                    Float32Regs:$z, Float32Regs:$lod),
3403               "tex.level.3d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3404               "[$t, \\{$x, $y, $z, $z\\}], $lod;",
3405               []>;
3406 def TEX_UNIFIED_3D_S32_F32_GRAD
3407   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3408                     Int32Regs:$b, Int32Regs:$a),
3409               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3410                    Float32Regs:$z,
3411                    Float32Regs:$gradx0, Float32Regs:$gradx1,
3412                    Float32Regs:$gradx2, Float32Regs:$grady0,
3413                    Float32Regs:$grady1, Float32Regs:$grady2),
3414               "tex.grad.3d.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3415               "[$t, \\{$x, $y, $z, $z\\}], "
3416               "\\{$gradx0, $gradx1, $gradx2, $gradx2\\}, "
3417               "\\{$grady0, $grady1, $grady2, $grady2\\};",
3418               []>;
3419 def TEX_UNIFIED_3D_U32_S32
3420   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3421                     Int32Regs:$b, Int32Regs:$a),
3422               (ins Int64Regs:$t, Int32Regs:$x, Int32Regs:$y,
3423                    Int32Regs:$z),
3424               "tex.3d.v4.u32.s32 \t\\{$r, $g, $b, $a\\}, "
3425               "[$t, \\{$x, $y, $z, $z\\}];",
3426               []>;
3427 def TEX_UNIFIED_3D_U32_F32
3428   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3429                     Int32Regs:$b, Int32Regs:$a),
3430               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3431                    Float32Regs:$z),
3432               "tex.3d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3433               "[$t, \\{$x, $y, $z, $z\\}];",
3434               []>;
3435 def TEX_UNIFIED_3D_U32_F32_LEVEL
3436   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3437                     Int32Regs:$b, Int32Regs:$a),
3438               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3439                    Float32Regs:$z, Float32Regs:$lod),
3440               "tex.level.3d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3441               "[$t, \\{$x, $y, $z, $z\\}], $lod;",
3442               []>;
3443 def TEX_UNIFIED_3D_U32_F32_GRAD
3444   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3445                     Int32Regs:$b, Int32Regs:$a),
3446               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y,
3447                    Float32Regs:$z,
3448                    Float32Regs:$gradx0, Float32Regs:$gradx1,
3449                    Float32Regs:$gradx2, Float32Regs:$grady0,
3450                    Float32Regs:$grady1, Float32Regs:$grady2),
3451               "tex.grad.3d.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3452               "[$t, \\{$x, $y, $z, $z\\}], "
3453               "\\{$gradx0, $gradx1, $gradx2, $gradx2\\}, "
3454               "\\{$grady0, $grady1, $grady2, $grady2\\};",
3455               []>;
3456
3457 def TEX_UNIFIED_CUBE_F32_F32
3458   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3459                     Float32Regs:$b, Float32Regs:$a),
3460               (ins Int64Regs:$t,
3461                Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
3462               "tex.cube.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3463               "[$t, \\{$x, $y, $z, $z\\}];",
3464               []>;
3465 def TEX_UNIFIED_CUBE_F32_F32_LEVEL
3466   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3467                     Float32Regs:$b, Float32Regs:$a),
3468               (ins Int64Regs:$t,
3469                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
3470                    Float32Regs:$lod),
3471               "tex.level.cube.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3472               "[$t, \\{$x, $y, $z, $z\\}], $lod;",
3473               []>;
3474 def TEX_UNIFIED_CUBE_S32_F32
3475   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3476                     Int32Regs:$b, Int32Regs:$a),
3477               (ins Int64Regs:$t,
3478                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
3479               "tex.cube.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3480               "[$t, \\{$x, $y, $z, $z\\}];",
3481               []>;
3482 def TEX_UNIFIED_CUBE_S32_F32_LEVEL
3483   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3484                     Int32Regs:$b, Int32Regs:$a),
3485               (ins Int64Regs:$t,
3486                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
3487                    Float32Regs:$lod),
3488               "tex.level.cube.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3489               "[$t, \\{$x, $y, $z, $z\\}], $lod;",
3490               []>;
3491 def TEX_UNIFIED_CUBE_U32_F32
3492   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3493                     Int32Regs:$b, Int32Regs:$a),
3494               (ins Int64Regs:$t,
3495                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
3496               "tex.cube.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3497               "[$t, \\{$x, $y, $z, $z\\}];",
3498               []>;
3499 def TEX_UNIFIED_CUBE_U32_F32_LEVEL
3500   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3501                     Int32Regs:$b, Int32Regs:$a),
3502               (ins Int64Regs:$t,
3503                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
3504                    Float32Regs:$lod),
3505               "tex.level.cube.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3506               "[$t, \\{$x, $y, $z, $z\\}], $lod;",
3507               []>;
3508
3509 def TEX_UNIFIED_CUBE_ARRAY_F32_F32
3510   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3511                     Float32Regs:$b, Float32Regs:$a),
3512               (ins Int64Regs:$t, Int32Regs:$l,
3513                Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
3514               "tex.acube.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3515               "[$t, \\{$l, $x, $y, $z\\}];",
3516               []>;
3517 def TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL
3518   : NVPTXInst<(outs Float32Regs:$r, Float32Regs:$g,
3519                     Float32Regs:$b, Float32Regs:$a),
3520               (ins Int64Regs:$t, Int32Regs:$l,
3521                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
3522                    Float32Regs:$lod),
3523               "tex.level.acube.v4.f32.f32 \t\\{$r, $g, $b, $a\\}, "
3524               "[$t, \\{$l, $x, $y, $z\\}], $lod;",
3525               []>;
3526 def TEX_UNIFIED_CUBE_ARRAY_S32_F32
3527   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3528                     Int32Regs:$b, Int32Regs:$a),
3529               (ins Int64Regs:$t, Int32Regs:$l,
3530                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
3531               "tex.acube.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3532               "[$t, \\{$l, $x, $y, $z\\}];",
3533               []>;
3534 def TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL
3535   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3536                     Int32Regs:$b, Int32Regs:$a),
3537               (ins Int64Regs:$t, Int32Regs:$l,
3538                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
3539                    Float32Regs:$lod),
3540               "tex.level.acube.v4.s32.f32 \t\\{$r, $g, $b, $a\\}, "
3541               "[$t, \\{$l, $x, $y, $z\\}], $lod;",
3542               []>;
3543 def TEX_UNIFIED_CUBE_ARRAY_U32_F32
3544   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3545                     Int32Regs:$b, Int32Regs:$a),
3546               (ins Int64Regs:$t, Int32Regs:$l,
3547                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z),
3548               "tex.acube.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3549               "[$t, \\{$l, $x, $y, $z\\}];",
3550               []>;
3551 def TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL
3552   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g,
3553                     Int32Regs:$b, Int32Regs:$a),
3554               (ins Int64Regs:$t, Int32Regs:$l,
3555                    Float32Regs:$x, Float32Regs:$y, Float32Regs:$z,
3556                    Float32Regs:$lod),
3557               "tex.level.acube.v4.u32.f32 \t\\{$r, $g, $b, $a\\}, "
3558               "[$t, \\{$l, $x, $y, $z\\}], $lod;",
3559               []>;
3560
3561 def TLD4_UNIFIED_R_2D_F32_F32
3562   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
3563                     Float32Regs:$v2, Float32Regs:$v3),
3564               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3565               "tld4.r.2d.v4.f32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
3566               "[$t, \\{$x, $y\\}];",
3567               []>;
3568 def TLD4_UNIFIED_G_2D_F32_F32
3569   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
3570                     Float32Regs:$v2, Float32Regs:$v3),
3571               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3572               "tld4.g.2d.v4.f32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
3573               "[$t, \\{$x, $y\\}];",
3574               []>;
3575 def TLD4_UNIFIED_B_2D_F32_F32
3576   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
3577                     Float32Regs:$v2, Float32Regs:$v3),
3578               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3579               "tld4.b.2d.v4.f32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
3580               "[$t, \\{$x, $y\\}];",
3581               []>;
3582 def TLD4_UNIFIED_A_2D_F32_F32
3583   : NVPTXInst<(outs Float32Regs:$v0, Float32Regs:$v1,
3584                     Float32Regs:$v2, Float32Regs:$v3),
3585               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3586               "tld4.a.2d.v4.f32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
3587               "[$t, \\{$x, $y\\}];",
3588               []>;
3589 def TLD4_UNIFIED_R_2D_S32_F32
3590   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3591                     Int32Regs:$v2, Int32Regs:$v3),
3592               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3593               "tld4.r.2d.v4.s32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
3594               "[$t, \\{$x, $y\\}];",
3595               []>;
3596 def TLD4_UNIFIED_G_2D_S32_F32
3597   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3598                     Int32Regs:$v2, Int32Regs:$v3),
3599               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3600               "tld4.g.2d.v4.s32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
3601               "[$t, \\{$x, $y\\}];",
3602               []>;
3603 def TLD4_UNIFIED_B_2D_S32_F32
3604   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3605                     Int32Regs:$v2, Int32Regs:$v3),
3606               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3607               "tld4.b.2d.v4.s32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
3608               "[$t, \\{$x, $y\\}];",
3609               []>;
3610 def TLD4_UNIFIED_A_2D_S32_F32
3611   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3612                     Int32Regs:$v2, Int32Regs:$v3),
3613               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3614               "tld4.a.2d.v4.s32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
3615               "[$t, \\{$x, $y\\}];",
3616               []>;
3617 def TLD4_UNIFIED_R_2D_U32_F32
3618   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3619                     Int32Regs:$v2, Int32Regs:$v3),
3620               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3621               "tld4.r.2d.v4.u32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
3622               "[$t, \\{$x, $y\\}];",
3623               []>;
3624 def TLD4_UNIFIED_G_2D_U32_F32
3625   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3626                     Int32Regs:$v2, Int32Regs:$v3),
3627               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3628               "tld4.g.2d.v4.u32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
3629               "[$t, \\{$x, $y\\}];",
3630               []>;
3631 def TLD4_UNIFIED_B_2D_U32_F32
3632   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3633                     Int32Regs:$v2, Int32Regs:$v3),
3634               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3635               "tld4.b.2d.v4.u32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
3636               "[$t, \\{$x, $y\\}];",
3637               []>;
3638 def TLD4_UNIFIED_A_2D_U32_F32
3639   : NVPTXInst<(outs Int32Regs:$v0, Int32Regs:$v1,
3640                     Int32Regs:$v2, Int32Regs:$v3),
3641               (ins Int64Regs:$t, Float32Regs:$x, Float32Regs:$y),
3642               "tld4.a.2d.v4.u32.f32 \t\\{$v0, $v1, $v2, $v3\\}, "
3643               "[$t, \\{$x, $y\\}];",
3644               []>;
3645 }
3646
3647
3648
3649 //=== Surface load instructions
3650 // .clamp variant
3651 let IsSuld = 1 in {
3652 def SULD_1D_I8_CLAMP
3653   : NVPTXInst<(outs Int16Regs:$r),
3654               (ins Int64Regs:$s, Int32Regs:$x),
3655               "suld.b.1d.b8.clamp \\{$r\\}, [$s, \\{$x\\}];",
3656               []>;
3657 def SULD_1D_I16_CLAMP
3658   : NVPTXInst<(outs Int16Regs:$r),
3659               (ins Int64Regs:$s, Int32Regs:$x),
3660               "suld.b.1d.b16.clamp \\{$r\\}, [$s, \\{$x\\}];",
3661               []>;
3662 def SULD_1D_I32_CLAMP
3663   : NVPTXInst<(outs Int32Regs:$r),
3664               (ins Int64Regs:$s, Int32Regs:$x),
3665               "suld.b.1d.b32.clamp \\{$r\\}, [$s, \\{$x\\}];",
3666               []>;
3667 def SULD_1D_I64_CLAMP
3668   : NVPTXInst<(outs Int64Regs:$r),
3669               (ins Int64Regs:$s, Int32Regs:$x),
3670               "suld.b.1d.b64.clamp \\{$r\\}, [$s, \\{$x\\}];",
3671               []>;
3672
3673 def SULD_1D_ARRAY_I8_CLAMP
3674   : NVPTXInst<(outs Int16Regs:$r),
3675               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3676               "suld.b.a1d.b8.clamp \\{$r\\}, [$s, \\{$l, $x\\}];",
3677               []>;
3678 def SULD_1D_ARRAY_I16_CLAMP
3679   : NVPTXInst<(outs Int16Regs:$r),
3680               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3681               "suld.b.a1d.b16.clamp \\{$r\\}, [$s, \\{$l, $x\\}];",
3682               []>;
3683 def SULD_1D_ARRAY_I32_CLAMP
3684   : NVPTXInst<(outs Int32Regs:$r),
3685               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3686               "suld.b.a1d.b32.clamp \\{$r\\}, [$s, \\{$l, $x\\}];",
3687               []>;
3688 def SULD_1D_ARRAY_I64_CLAMP
3689   : NVPTXInst<(outs Int64Regs:$r),
3690               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3691               "suld.b.a1d.b64.clamp \\{$r\\}, [$s, \\{$l, $x\\}];",
3692               []>;
3693
3694 def SULD_2D_I8_CLAMP
3695   : NVPTXInst<(outs Int16Regs:$r),
3696               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3697               "suld.b.2d.b8.clamp \\{$r\\}, [$s, \\{$x, $y\\}];",
3698               []>;
3699 def SULD_2D_I16_CLAMP
3700   : NVPTXInst<(outs Int16Regs:$r),
3701               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3702               "suld.b.2d.b16.clamp \\{$r\\}, [$s, \\{$x, $y\\}];",
3703               []>;
3704 def SULD_2D_I32_CLAMP
3705   : NVPTXInst<(outs Int32Regs:$r),
3706               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3707               "suld.b.2d.b32.clamp \\{$r\\}, [$s, \\{$x, $y\\}];",
3708               []>;
3709 def SULD_2D_I64_CLAMP
3710   : NVPTXInst<(outs Int64Regs:$r),
3711               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3712               "suld.b.2d.b64.clamp \\{$r\\}, [$s, \\{$x, $y\\}];",
3713               []>;
3714
3715 def SULD_2D_ARRAY_I8_CLAMP
3716   : NVPTXInst<(outs Int16Regs:$r),
3717               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3718               "suld.b.a2d.b8.clamp \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
3719               []>;
3720 def SULD_2D_ARRAY_I16_CLAMP
3721   : NVPTXInst<(outs Int16Regs:$r),
3722               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3723               "suld.b.a2d.b16.clamp \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
3724               []>;
3725 def SULD_2D_ARRAY_I32_CLAMP
3726   : NVPTXInst<(outs Int32Regs:$r),
3727               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3728               "suld.b.a2d.b32.clamp \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
3729               []>;
3730 def SULD_2D_ARRAY_I64_CLAMP
3731   : NVPTXInst<(outs Int64Regs:$r),
3732               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3733               "suld.b.a2d.b64.clamp \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
3734               []>;
3735
3736 def SULD_3D_I8_CLAMP
3737   : NVPTXInst<(outs Int16Regs:$r),
3738               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3739               "suld.b.3d.b8.clamp \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
3740               []>;
3741 def SULD_3D_I16_CLAMP
3742   : NVPTXInst<(outs Int16Regs:$r),
3743               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3744               "suld.b.3d.b16.clamp \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
3745               []>;
3746 def SULD_3D_I32_CLAMP
3747   : NVPTXInst<(outs Int32Regs:$r),
3748               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3749               "suld.b.3d.b32.clamp \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
3750               []>;
3751 def SULD_3D_I64_CLAMP
3752   : NVPTXInst<(outs Int64Regs:$r),
3753               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3754               "suld.b.3d.b64.clamp \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
3755               []>;
3756 }
3757
3758 let IsSuld = 2 in {
3759 def SULD_1D_V2I8_CLAMP
3760   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3761               (ins Int64Regs:$s, Int32Regs:$x),
3762               "suld.b.1d.v2.b8.clamp \\{$r, $g\\}, [$s, \\{$x\\}];",
3763               []>;
3764 def SULD_1D_V2I16_CLAMP
3765   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3766               (ins Int64Regs:$s, Int32Regs:$x),
3767               "suld.b.1d.v2.b16.clamp \\{$r, $g\\}, [$s, \\{$x\\}];",
3768               []>;
3769 def SULD_1D_V2I32_CLAMP
3770   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3771               (ins Int64Regs:$s, Int32Regs:$x),
3772               "suld.b.1d.v2.b32.clamp \\{$r, $g\\}, [$s, \\{$x\\}];",
3773               []>;
3774 def SULD_1D_V2I64_CLAMP
3775   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3776               (ins Int64Regs:$s, Int32Regs:$x),
3777               "suld.b.1d.v2.b64.clamp \\{$r, $g\\}, [$s, \\{$x\\}];",
3778               []>;
3779
3780 def SULD_1D_ARRAY_V2I8_CLAMP
3781   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3782               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3783               "suld.b.a1d.v2.b8.clamp \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
3784               []>;
3785 def SULD_1D_ARRAY_V2I16_CLAMP
3786   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3787               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3788               "suld.b.a1d.v2.b16.clamp \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
3789               []>;
3790 def SULD_1D_ARRAY_V2I32_CLAMP
3791   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3792               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3793               "suld.b.a1d.v2.b32.clamp \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
3794               []>;
3795 def SULD_1D_ARRAY_V2I64_CLAMP
3796   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3797               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3798               "suld.b.a1d.v2.b64.clamp \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
3799               []>;
3800
3801 def SULD_2D_V2I8_CLAMP
3802   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3803               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3804               "suld.b.2d.v2.b8.clamp \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
3805               []>;
3806 def SULD_2D_V2I16_CLAMP
3807   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3808               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3809               "suld.b.2d.v2.b16.clamp \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
3810               []>;
3811 def SULD_2D_V2I32_CLAMP
3812   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3813               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3814               "suld.b.2d.v2.b32.clamp \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
3815               []>;
3816 def SULD_2D_V2I64_CLAMP
3817   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3818               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3819               "suld.b.2d.v2.b64.clamp \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
3820               []>;
3821
3822 def SULD_2D_ARRAY_V2I8_CLAMP
3823   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3824               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3825               "suld.b.a2d.v2.b8.clamp \\{$r, $g\\}, "
3826               "[$s, \\{$l, $x, $y, $y\\}];",
3827               []>;
3828 def SULD_2D_ARRAY_V2I16_CLAMP
3829   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3830               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3831               "suld.b.a2d.v2.b16.clamp \\{$r, $g\\}, "
3832               "[$s, \\{$l, $x, $y, $y\\}];",
3833               []>;
3834 def SULD_2D_ARRAY_V2I32_CLAMP
3835   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3836               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3837               "suld.b.a2d.v2.b32.clamp \\{$r, $g\\}, "
3838               "[$s, \\{$l, $x, $y, $y\\}];",
3839               []>;
3840 def SULD_2D_ARRAY_V2I64_CLAMP
3841   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3842               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3843               "suld.b.a2d.v2.b64.clamp \\{$r, $g\\}, "
3844               "[$s, \\{$l, $x, $y, $y\\}];",
3845               []>;
3846
3847 def SULD_3D_V2I8_CLAMP
3848   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3849               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3850               "suld.b.3d.v2.b8.clamp \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
3851               []>;
3852 def SULD_3D_V2I16_CLAMP
3853   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
3854               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3855               "suld.b.3d.v2.b16.clamp \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
3856               []>;
3857 def SULD_3D_V2I32_CLAMP
3858   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
3859               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3860               "suld.b.3d.v2.b32.clamp \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
3861               []>;
3862 def SULD_3D_V2I64_CLAMP
3863   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
3864               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3865               "suld.b.3d.v2.b64.clamp \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
3866               []>;
3867 }
3868
3869 let IsSuld = 3 in {
3870 def SULD_1D_V4I8_CLAMP
3871   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3872               (ins Int64Regs:$s, Int32Regs:$x),
3873               "suld.b.1d.v4.b8.clamp \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
3874               []>;
3875 def SULD_1D_V4I16_CLAMP
3876   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3877               (ins Int64Regs:$s, Int32Regs:$x),
3878               "suld.b.1d.v4.b16.clamp \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
3879               []>;
3880 def SULD_1D_V4I32_CLAMP
3881   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3882               (ins Int64Regs:$s, Int32Regs:$x),
3883               "suld.b.1d.v4.b32.clamp \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
3884               []>;
3885
3886 def SULD_1D_ARRAY_V4I8_CLAMP
3887   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3888               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3889               "suld.b.a1d.v4.b8.clamp \\{$r, $g, $b, $a\\}, "
3890               "[$s, \\{$l, $x\\}];",
3891               []>;
3892 def SULD_1D_ARRAY_V4I16_CLAMP
3893   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3894               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3895               "suld.b.a1d.v4.b16.clamp \\{$r, $g, $b, $a\\}, "
3896               "[$s, \\{$l, $x\\}];",
3897               []>;
3898 def SULD_1D_ARRAY_V4I32_CLAMP
3899   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3900               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3901               "suld.b.a1d.v4.b32.clamp \\{$r, $g, $b, $a\\}, "
3902               "[$s, \\{$l, $x\\}];",
3903               []>;
3904
3905 def SULD_2D_V4I8_CLAMP
3906   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3907               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3908               "suld.b.2d.v4.b8.clamp \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
3909               []>;
3910 def SULD_2D_V4I16_CLAMP
3911   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3912               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3913               "suld.b.2d.v4.b16.clamp \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
3914               []>;
3915 def SULD_2D_V4I32_CLAMP
3916   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3917               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
3918               "suld.b.2d.v4.b32.clamp \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
3919               []>;
3920
3921 def SULD_2D_ARRAY_V4I8_CLAMP
3922   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3923               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3924               "suld.b.a2d.v4.b8.clamp \\{$r, $g, $b, $a\\}, "
3925               "[$s, \\{$l, $x, $y, $y\\}];",
3926               []>;
3927 def SULD_2D_ARRAY_V4I16_CLAMP
3928   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3929               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3930               "suld.b.a2d.v4.b16.clamp \\{$r, $g, $b, $a\\}, "
3931               "[$s, \\{$l, $x, $y, $y\\}];",
3932               []>;
3933 def SULD_2D_ARRAY_V4I32_CLAMP
3934   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3935               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
3936               "suld.b.a2d.v4.b32.clamp \\{$r, $g, $b, $a\\}, "
3937               "[$s, \\{$l, $x, $y, $y\\}];",
3938               []>;
3939
3940
3941 def SULD_3D_V4I8_CLAMP
3942   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3943               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3944               "suld.b.3d.v4.b8.clamp \\{$r, $g, $b, $a\\}, "
3945               "[$s, \\{$x, $y, $z, $z\\}];",
3946               []>;
3947 def SULD_3D_V4I16_CLAMP
3948   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
3949               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3950               "suld.b.3d.v4.b16.clamp \\{$r, $g, $b, $a\\}, "
3951               "[$s, \\{$x, $y, $z, $z\\}];",
3952               []>;
3953 def SULD_3D_V4I32_CLAMP
3954   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
3955               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
3956               "suld.b.3d.v4.b32.clamp \\{$r, $g, $b, $a\\}, "
3957               "[$s, \\{$x, $y, $z, $z\\}];",
3958               []>;
3959 }
3960
3961
3962 // .trap variant
3963 let IsSuld = 1 in {
3964 def SULD_1D_I8_TRAP
3965   : NVPTXInst<(outs Int16Regs:$r),
3966               (ins Int64Regs:$s, Int32Regs:$x),
3967               "suld.b.1d.b8.trap \\{$r\\}, [$s, \\{$x\\}];",
3968               []>;
3969 def SULD_1D_I16_TRAP
3970   : NVPTXInst<(outs Int16Regs:$r),
3971               (ins Int64Regs:$s, Int32Regs:$x),
3972               "suld.b.1d.b16.trap \\{$r\\}, [$s, \\{$x\\}];",
3973               []>;
3974 def SULD_1D_I32_TRAP
3975   : NVPTXInst<(outs Int32Regs:$r),
3976               (ins Int64Regs:$s, Int32Regs:$x),
3977               "suld.b.1d.b32.trap \\{$r\\}, [$s, \\{$x\\}];",
3978               []>;
3979 def SULD_1D_I64_TRAP
3980   : NVPTXInst<(outs Int64Regs:$r),
3981               (ins Int64Regs:$s, Int32Regs:$x),
3982               "suld.b.1d.b64.trap \\{$r\\}, [$s, \\{$x\\}];",
3983               []>;
3984
3985 def SULD_1D_ARRAY_I8_TRAP
3986   : NVPTXInst<(outs Int16Regs:$r),
3987               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3988               "suld.b.a1d.b8.trap \\{$r\\}, [$s, \\{$l, $x\\}];",
3989               []>;
3990 def SULD_1D_ARRAY_I16_TRAP
3991   : NVPTXInst<(outs Int16Regs:$r),
3992               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3993               "suld.b.a1d.b16.trap \\{$r\\}, [$s, \\{$l, $x\\}];",
3994               []>;
3995 def SULD_1D_ARRAY_I32_TRAP
3996   : NVPTXInst<(outs Int32Regs:$r),
3997               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
3998               "suld.b.a1d.b32.trap \\{$r\\}, [$s, \\{$l, $x\\}];",
3999               []>;
4000 def SULD_1D_ARRAY_I64_TRAP
4001   : NVPTXInst<(outs Int64Regs:$r),
4002               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4003               "suld.b.a1d.b64.trap \\{$r\\}, [$s, \\{$l, $x\\}];",
4004               []>;
4005
4006 def SULD_2D_I8_TRAP
4007   : NVPTXInst<(outs Int16Regs:$r),
4008               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4009               "suld.b.2d.b8.trap \\{$r\\}, [$s, \\{$x, $y\\}];",
4010               []>;
4011 def SULD_2D_I16_TRAP
4012   : NVPTXInst<(outs Int16Regs:$r),
4013               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4014               "suld.b.2d.b16.trap \\{$r\\}, [$s, \\{$x, $y\\}];",
4015               []>;
4016 def SULD_2D_I32_TRAP
4017   : NVPTXInst<(outs Int32Regs:$r),
4018               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4019               "suld.b.2d.b32.trap \\{$r\\}, [$s, \\{$x, $y\\}];",
4020               []>;
4021 def SULD_2D_I64_TRAP
4022   : NVPTXInst<(outs Int64Regs:$r),
4023               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4024               "suld.b.2d.b64.trap \\{$r\\}, [$s, \\{$x, $y\\}];",
4025               []>;
4026
4027 def SULD_2D_ARRAY_I8_TRAP
4028   : NVPTXInst<(outs Int16Regs:$r),
4029               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4030               "suld.b.a2d.b8.trap \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
4031               []>;
4032 def SULD_2D_ARRAY_I16_TRAP
4033   : NVPTXInst<(outs Int16Regs:$r),
4034               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4035               "suld.b.a2d.b16.trap \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
4036               []>;
4037 def SULD_2D_ARRAY_I32_TRAP
4038   : NVPTXInst<(outs Int32Regs:$r),
4039               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4040               "suld.b.a2d.b32.trap \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
4041               []>;
4042 def SULD_2D_ARRAY_I64_TRAP
4043   : NVPTXInst<(outs Int64Regs:$r),
4044               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4045               "suld.b.a2d.b64.trap \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
4046               []>;
4047
4048 def SULD_3D_I8_TRAP
4049   : NVPTXInst<(outs Int16Regs:$r),
4050               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4051               "suld.b.3d.b8.trap \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
4052               []>;
4053 def SULD_3D_I16_TRAP
4054   : NVPTXInst<(outs Int16Regs:$r),
4055               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4056               "suld.b.3d.b16.trap \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
4057               []>;
4058 def SULD_3D_I32_TRAP
4059   : NVPTXInst<(outs Int32Regs:$r),
4060               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4061               "suld.b.3d.b32.trap \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
4062               []>;
4063 def SULD_3D_I64_TRAP
4064   : NVPTXInst<(outs Int64Regs:$r),
4065               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4066               "suld.b.3d.b64.trap \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
4067               []>;
4068 }
4069
4070 let IsSuld = 2 in {
4071 def SULD_1D_V2I8_TRAP
4072   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4073               (ins Int64Regs:$s, Int32Regs:$x),
4074               "suld.b.1d.v2.b8.trap \\{$r, $g\\}, [$s, \\{$x\\}];",
4075               []>;
4076 def SULD_1D_V2I16_TRAP
4077   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4078               (ins Int64Regs:$s, Int32Regs:$x),
4079               "suld.b.1d.v2.b16.trap \\{$r, $g\\}, [$s, \\{$x\\}];",
4080               []>;
4081 def SULD_1D_V2I32_TRAP
4082   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4083               (ins Int64Regs:$s, Int32Regs:$x),
4084               "suld.b.1d.v2.b32.trap \\{$r, $g\\}, [$s, \\{$x\\}];",
4085               []>;
4086 def SULD_1D_V2I64_TRAP
4087   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4088               (ins Int64Regs:$s, Int32Regs:$x),
4089               "suld.b.1d.v2.b64.trap \\{$r, $g\\}, [$s, \\{$x\\}];",
4090               []>;
4091
4092 def SULD_1D_ARRAY_V2I8_TRAP
4093   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4094               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4095               "suld.b.a1d.v2.b8.trap \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
4096               []>;
4097 def SULD_1D_ARRAY_V2I16_TRAP
4098   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4099               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4100               "suld.b.a1d.v2.b16.trap \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
4101               []>;
4102 def SULD_1D_ARRAY_V2I32_TRAP
4103   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4104               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4105               "suld.b.a1d.v2.b32.trap \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
4106               []>;
4107 def SULD_1D_ARRAY_V2I64_TRAP
4108   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4109               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4110               "suld.b.a1d.v2.b64.trap \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
4111               []>;
4112
4113 def SULD_2D_V2I8_TRAP
4114   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4115               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4116               "suld.b.2d.v2.b8.trap \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
4117               []>;
4118 def SULD_2D_V2I16_TRAP
4119   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4120               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4121               "suld.b.2d.v2.b16.trap \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
4122               []>;
4123 def SULD_2D_V2I32_TRAP
4124   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4125               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4126               "suld.b.2d.v2.b32.trap \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
4127               []>;
4128 def SULD_2D_V2I64_TRAP
4129   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4130               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4131               "suld.b.2d.v2.b64.trap \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
4132               []>;
4133
4134 def SULD_2D_ARRAY_V2I8_TRAP
4135   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4136               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4137               "suld.b.a2d.v2.b8.trap \\{$r, $g\\}, "
4138               "[$s, \\{$l, $x, $y, $y\\}];",
4139               []>;
4140 def SULD_2D_ARRAY_V2I16_TRAP
4141   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4142               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4143               "suld.b.a2d.v2.b16.trap \\{$r, $g\\}, "
4144               "[$s, \\{$l, $x, $y, $y\\}];",
4145               []>;
4146 def SULD_2D_ARRAY_V2I32_TRAP
4147   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4148               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4149               "suld.b.a2d.v2.b32.trap \\{$r, $g\\}, "
4150               "[$s, \\{$l, $x, $y, $y\\}];",
4151               []>;
4152 def SULD_2D_ARRAY_V2I64_TRAP
4153   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4154               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4155               "suld.b.a2d.v2.b64.trap \\{$r, $g\\}, "
4156               "[$s, \\{$l, $x, $y, $y\\}];",
4157               []>;
4158
4159 def SULD_3D_V2I8_TRAP
4160   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4161               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4162               "suld.b.3d.v2.b8.trap \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
4163               []>;
4164 def SULD_3D_V2I16_TRAP
4165   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4166               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4167               "suld.b.3d.v2.b16.trap \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
4168               []>;
4169 def SULD_3D_V2I32_TRAP
4170   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4171               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4172               "suld.b.3d.v2.b32.trap \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
4173               []>;
4174 def SULD_3D_V2I64_TRAP
4175   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4176               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4177               "suld.b.3d.v2.b64.trap \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
4178               []>;
4179 }
4180
4181 let IsSuld = 3 in {
4182 def SULD_1D_V4I8_TRAP
4183   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4184               (ins Int64Regs:$s, Int32Regs:$x),
4185               "suld.b.1d.v4.b8.trap \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
4186               []>;
4187 def SULD_1D_V4I16_TRAP
4188   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4189               (ins Int64Regs:$s, Int32Regs:$x),
4190               "suld.b.1d.v4.b16.trap \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
4191               []>;
4192 def SULD_1D_V4I32_TRAP
4193   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4194               (ins Int64Regs:$s, Int32Regs:$x),
4195               "suld.b.1d.v4.b32.trap \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
4196               []>;
4197
4198 def SULD_1D_ARRAY_V4I8_TRAP
4199   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4200               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4201               "suld.b.a1d.v4.b8.trap \\{$r, $g, $b, $a\\}, "
4202               "[$s, \\{$l, $x\\}];",
4203               []>;
4204 def SULD_1D_ARRAY_V4I16_TRAP
4205   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4206               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4207               "suld.b.a1d.v4.b16.trap \\{$r, $g, $b, $a\\}, "
4208               "[$s, \\{$l, $x\\}];",
4209               []>;
4210 def SULD_1D_ARRAY_V4I32_TRAP
4211   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4212               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4213               "suld.b.a1d.v4.b32.trap \\{$r, $g, $b, $a\\}, "
4214               "[$s, \\{$l, $x\\}];",
4215               []>;
4216
4217 def SULD_2D_V4I8_TRAP
4218   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4219               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4220               "suld.b.2d.v4.b8.trap \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
4221               []>;
4222 def SULD_2D_V4I16_TRAP
4223   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4224               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4225               "suld.b.2d.v4.b16.trap \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
4226               []>;
4227 def SULD_2D_V4I32_TRAP
4228   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4229               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4230               "suld.b.2d.v4.b32.trap \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
4231               []>;
4232
4233 def SULD_2D_ARRAY_V4I8_TRAP
4234   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4235               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4236               "suld.b.a2d.v4.b8.trap \\{$r, $g, $b, $a\\}, "
4237               "[$s, \\{$l, $x, $y, $y\\}];",
4238               []>;
4239 def SULD_2D_ARRAY_V4I16_TRAP
4240   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4241               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4242               "suld.b.a2d.v4.b16.trap \\{$r, $g, $b, $a\\}, "
4243               "[$s, \\{$l, $x, $y, $y\\}];",
4244               []>;
4245 def SULD_2D_ARRAY_V4I32_TRAP
4246   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4247               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4248               "suld.b.a2d.v4.b32.trap \\{$r, $g, $b, $a\\}, "
4249               "[$s, \\{$l, $x, $y, $y\\}];",
4250               []>;
4251
4252
4253 def SULD_3D_V4I8_TRAP
4254   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4255               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4256               "suld.b.3d.v4.b8.trap \\{$r, $g, $b, $a\\}, "
4257               "[$s, \\{$x, $y, $z, $z\\}];",
4258               []>;
4259 def SULD_3D_V4I16_TRAP
4260   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4261               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4262               "suld.b.3d.v4.b16.trap \\{$r, $g, $b, $a\\}, "
4263               "[$s, \\{$x, $y, $z, $z\\}];",
4264               []>;
4265 def SULD_3D_V4I32_TRAP
4266   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4267               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4268               "suld.b.3d.v4.b32.trap \\{$r, $g, $b, $a\\}, "
4269               "[$s, \\{$x, $y, $z, $z\\}];",
4270               []>;
4271 }
4272
4273 // .zero variant
4274 let IsSuld = 1 in {
4275 def SULD_1D_I8_ZERO
4276   : NVPTXInst<(outs Int16Regs:$r),
4277               (ins Int64Regs:$s, Int32Regs:$x),
4278               "suld.b.1d.b8.zero \\{$r\\}, [$s, \\{$x\\}];",
4279               []>;
4280 def SULD_1D_I16_ZERO
4281   : NVPTXInst<(outs Int16Regs:$r),
4282               (ins Int64Regs:$s, Int32Regs:$x),
4283               "suld.b.1d.b16.zero \\{$r\\}, [$s, \\{$x\\}];",
4284               []>;
4285 def SULD_1D_I32_ZERO
4286   : NVPTXInst<(outs Int32Regs:$r),
4287               (ins Int64Regs:$s, Int32Regs:$x),
4288               "suld.b.1d.b32.zero \\{$r\\}, [$s, \\{$x\\}];",
4289               []>;
4290 def SULD_1D_I64_ZERO
4291   : NVPTXInst<(outs Int64Regs:$r),
4292               (ins Int64Regs:$s, Int32Regs:$x),
4293               "suld.b.1d.b64.zero \\{$r\\}, [$s, \\{$x\\}];",
4294               []>;
4295
4296 def SULD_1D_ARRAY_I8_ZERO
4297   : NVPTXInst<(outs Int16Regs:$r),
4298               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4299               "suld.b.a1d.b8.zero \\{$r\\}, [$s, \\{$l, $x\\}];",
4300               []>;
4301 def SULD_1D_ARRAY_I16_ZERO
4302   : NVPTXInst<(outs Int16Regs:$r),
4303               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4304               "suld.b.a1d.b16.zero \\{$r\\}, [$s, \\{$l, $x\\}];",
4305               []>;
4306 def SULD_1D_ARRAY_I32_ZERO
4307   : NVPTXInst<(outs Int32Regs:$r),
4308               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4309               "suld.b.a1d.b32.zero \\{$r\\}, [$s, \\{$l, $x\\}];",
4310               []>;
4311 def SULD_1D_ARRAY_I64_ZERO
4312   : NVPTXInst<(outs Int64Regs:$r),
4313               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4314               "suld.b.a1d.b64.zero \\{$r\\}, [$s, \\{$l, $x\\}];",
4315               []>;
4316
4317 def SULD_2D_I8_ZERO
4318   : NVPTXInst<(outs Int16Regs:$r),
4319               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4320               "suld.b.2d.b8.zero \\{$r\\}, [$s, \\{$x, $y\\}];",
4321               []>;
4322 def SULD_2D_I16_ZERO
4323   : NVPTXInst<(outs Int16Regs:$r),
4324               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4325               "suld.b.2d.b16.zero \\{$r\\}, [$s, \\{$x, $y\\}];",
4326               []>;
4327 def SULD_2D_I32_ZERO
4328   : NVPTXInst<(outs Int32Regs:$r),
4329               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4330               "suld.b.2d.b32.zero \\{$r\\}, [$s, \\{$x, $y\\}];",
4331               []>;
4332 def SULD_2D_I64_ZERO
4333   : NVPTXInst<(outs Int64Regs:$r),
4334               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4335               "suld.b.2d.b64.zero \\{$r\\}, [$s, \\{$x, $y\\}];",
4336               []>;
4337
4338 def SULD_2D_ARRAY_I8_ZERO
4339   : NVPTXInst<(outs Int16Regs:$r),
4340               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4341               "suld.b.a2d.b8.zero \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
4342               []>;
4343 def SULD_2D_ARRAY_I16_ZERO
4344   : NVPTXInst<(outs Int16Regs:$r),
4345               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4346               "suld.b.a2d.b16.zero \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
4347               []>;
4348 def SULD_2D_ARRAY_I32_ZERO
4349   : NVPTXInst<(outs Int32Regs:$r),
4350               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4351               "suld.b.a2d.b32.zero \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
4352               []>;
4353 def SULD_2D_ARRAY_I64_ZERO
4354   : NVPTXInst<(outs Int64Regs:$r),
4355               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4356               "suld.b.a2d.b64.zero \\{$r\\}, [$s, \\{$l, $x, $y, $y\\}];",
4357               []>;
4358
4359 def SULD_3D_I8_ZERO
4360   : NVPTXInst<(outs Int16Regs:$r),
4361               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4362               "suld.b.3d.b8.zero \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
4363               []>;
4364 def SULD_3D_I16_ZERO
4365   : NVPTXInst<(outs Int16Regs:$r),
4366               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4367               "suld.b.3d.b16.zero \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
4368               []>;
4369 def SULD_3D_I32_ZERO
4370   : NVPTXInst<(outs Int32Regs:$r),
4371               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4372               "suld.b.3d.b32.zero \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
4373               []>;
4374 def SULD_3D_I64_ZERO
4375   : NVPTXInst<(outs Int64Regs:$r),
4376               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4377               "suld.b.3d.b64.zero \\{$r\\}, [$s, \\{$x, $y, $z, $z\\}];",
4378               []>;
4379 }
4380
4381 let IsSuld = 2 in {
4382 def SULD_1D_V2I8_ZERO
4383   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4384               (ins Int64Regs:$s, Int32Regs:$x),
4385               "suld.b.1d.v2.b8.zero \\{$r, $g\\}, [$s, \\{$x\\}];",
4386               []>;
4387 def SULD_1D_V2I16_ZERO
4388   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4389               (ins Int64Regs:$s, Int32Regs:$x),
4390               "suld.b.1d.v2.b16.zero \\{$r, $g\\}, [$s, \\{$x\\}];",
4391               []>;
4392 def SULD_1D_V2I32_ZERO
4393   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4394               (ins Int64Regs:$s, Int32Regs:$x),
4395               "suld.b.1d.v2.b32.zero \\{$r, $g\\}, [$s, \\{$x\\}];",
4396               []>;
4397 def SULD_1D_V2I64_ZERO
4398   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4399               (ins Int64Regs:$s, Int32Regs:$x),
4400               "suld.b.1d.v2.b64.zero \\{$r, $g\\}, [$s, \\{$x\\}];",
4401               []>;
4402
4403 def SULD_1D_ARRAY_V2I8_ZERO
4404   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4405               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4406               "suld.b.a1d.v2.b8.zero \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
4407               []>;
4408 def SULD_1D_ARRAY_V2I16_ZERO
4409   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4410               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4411               "suld.b.a1d.v2.b16.zero \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
4412               []>;
4413 def SULD_1D_ARRAY_V2I32_ZERO
4414   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4415               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4416               "suld.b.a1d.v2.b32.zero \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
4417               []>;
4418 def SULD_1D_ARRAY_V2I64_ZERO
4419   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4420               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4421               "suld.b.a1d.v2.b64.zero \\{$r, $g\\}, [$s, \\{$l, $x\\}];",
4422               []>;
4423
4424 def SULD_2D_V2I8_ZERO
4425   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4426               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4427               "suld.b.2d.v2.b8.zero \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
4428               []>;
4429 def SULD_2D_V2I16_ZERO
4430   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4431               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4432               "suld.b.2d.v2.b16.zero \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
4433               []>;
4434 def SULD_2D_V2I32_ZERO
4435   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4436               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4437               "suld.b.2d.v2.b32.zero \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
4438               []>;
4439 def SULD_2D_V2I64_ZERO
4440   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4441               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4442               "suld.b.2d.v2.b64.zero \\{$r, $g\\}, [$s, \\{$x, $y\\}];",
4443               []>;
4444
4445 def SULD_2D_ARRAY_V2I8_ZERO
4446   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4447               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4448               "suld.b.a2d.v2.b8.zero \\{$r, $g\\}, "
4449               "[$s, \\{$l, $x, $y, $y\\}];",
4450               []>;
4451 def SULD_2D_ARRAY_V2I16_ZERO
4452   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4453               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4454               "suld.b.a2d.v2.b16.zero \\{$r, $g\\}, "
4455               "[$s, \\{$l, $x, $y, $y\\}];",
4456               []>;
4457 def SULD_2D_ARRAY_V2I32_ZERO
4458   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4459               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4460               "suld.b.a2d.v2.b32.zero \\{$r, $g\\}, "
4461               "[$s, \\{$l, $x, $y, $y\\}];",
4462               []>;
4463 def SULD_2D_ARRAY_V2I64_ZERO
4464   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4465               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4466               "suld.b.a2d.v2.b64.zero \\{$r, $g\\}, "
4467               "[$s, \\{$l, $x, $y, $y\\}];",
4468               []>;
4469
4470 def SULD_3D_V2I8_ZERO
4471   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4472               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4473               "suld.b.3d.v2.b8.zero \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
4474               []>;
4475 def SULD_3D_V2I16_ZERO
4476   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g),
4477               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4478               "suld.b.3d.v2.b16.zero \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
4479               []>;
4480 def SULD_3D_V2I32_ZERO
4481   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g),
4482               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4483               "suld.b.3d.v2.b32.zero \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
4484               []>;
4485 def SULD_3D_V2I64_ZERO
4486   : NVPTXInst<(outs Int64Regs:$r, Int64Regs:$g),
4487               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4488               "suld.b.3d.v2.b64.zero \\{$r, $g\\}, [$s, \\{$x, $y, $z, $z\\}];",
4489               []>;
4490 }
4491
4492 let IsSuld = 3 in {
4493 def SULD_1D_V4I8_ZERO
4494   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4495               (ins Int64Regs:$s, Int32Regs:$x),
4496               "suld.b.1d.v4.b8.zero \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
4497               []>;
4498 def SULD_1D_V4I16_ZERO
4499   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4500               (ins Int64Regs:$s, Int32Regs:$x),
4501               "suld.b.1d.v4.b16.zero \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
4502               []>;
4503 def SULD_1D_V4I32_ZERO
4504   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4505               (ins Int64Regs:$s, Int32Regs:$x),
4506               "suld.b.1d.v4.b32.zero \\{$r, $g, $b, $a\\}, [$s, \\{$x\\}];",
4507               []>;
4508
4509 def SULD_1D_ARRAY_V4I8_ZERO
4510   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4511               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4512               "suld.b.a1d.v4.b8.zero \\{$r, $g, $b, $a\\}, "
4513               "[$s, \\{$l, $x\\}];",
4514               []>;
4515 def SULD_1D_ARRAY_V4I16_ZERO
4516   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4517               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4518               "suld.b.a1d.v4.b16.zero \\{$r, $g, $b, $a\\}, "
4519               "[$s, \\{$l, $x\\}];",
4520               []>;
4521 def SULD_1D_ARRAY_V4I32_ZERO
4522   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4523               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x),
4524               "suld.b.a1d.v4.b32.zero \\{$r, $g, $b, $a\\}, "
4525               "[$s, \\{$l, $x\\}];",
4526               []>;
4527
4528 def SULD_2D_V4I8_ZERO
4529   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4530               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4531               "suld.b.2d.v4.b8.zero \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
4532               []>;
4533 def SULD_2D_V4I16_ZERO
4534   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4535               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4536               "suld.b.2d.v4.b16.zero \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
4537               []>;
4538 def SULD_2D_V4I32_ZERO
4539   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4540               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y),
4541               "suld.b.2d.v4.b32.zero \\{$r, $g, $b, $a\\}, [$s, \\{$x, $y\\}];",
4542               []>;
4543
4544 def SULD_2D_ARRAY_V4I8_ZERO
4545   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4546               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4547               "suld.b.a2d.v4.b8.zero \\{$r, $g, $b, $a\\}, "
4548               "[$s, \\{$l, $x, $y, $y\\}];",
4549               []>;
4550 def SULD_2D_ARRAY_V4I16_ZERO
4551   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4552               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4553               "suld.b.a2d.v4.b16.zero \\{$r, $g, $b, $a\\}, "
4554               "[$s, \\{$l, $x, $y, $y\\}];",
4555               []>;
4556 def SULD_2D_ARRAY_V4I32_ZERO
4557   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4558               (ins Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y),
4559               "suld.b.a2d.v4.b32.zero \\{$r, $g, $b, $a\\}, "
4560               "[$s, \\{$l, $x, $y, $y\\}];",
4561               []>;
4562
4563
4564 def SULD_3D_V4I8_ZERO
4565   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4566               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4567               "suld.b.3d.v4.b8.zero \\{$r, $g, $b, $a\\}, "
4568               "[$s, \\{$x, $y, $z, $z\\}];",
4569               []>;
4570 def SULD_3D_V4I16_ZERO
4571   : NVPTXInst<(outs Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4572               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4573               "suld.b.3d.v4.b16.zero \\{$r, $g, $b, $a\\}, "
4574               "[$s, \\{$x, $y, $z, $z\\}];",
4575               []>;
4576 def SULD_3D_V4I32_ZERO
4577   : NVPTXInst<(outs Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4578               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z),
4579               "suld.b.3d.v4.b32.zero \\{$r, $g, $b, $a\\}, "
4580               "[$s, \\{$x, $y, $z, $z\\}];",
4581               []>;
4582 }
4583
4584 //-----------------------------------
4585 // Texture Query Intrinsics
4586 //-----------------------------------
4587
4588 let IsSurfTexQuery = 1 in {
4589 def TXQ_CHANNEL_ORDER
4590   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4591               "txq.channel_order.b32 \t$d, [$a];",
4592               []>;
4593 def TXQ_CHANNEL_DATA_TYPE
4594   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4595               "txq.channel_data_type.b32 \t$d, [$a];",
4596               []>;
4597 def TXQ_WIDTH
4598   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4599               "txq.width.b32 \t$d, [$a];",
4600               []>;
4601 def TXQ_HEIGHT
4602   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4603               "txq.height.b32 \t$d, [$a];",
4604               []>;
4605 def TXQ_DEPTH
4606   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4607               "txq.depth.b32 \t$d, [$a];",
4608               []>;
4609 def TXQ_ARRAY_SIZE
4610   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4611               "txq.array_size.b32 \t$d, [$a];",
4612               []>;
4613 def TXQ_NUM_SAMPLES
4614   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4615               "txq.num_samples.b32 \t$d, [$a];",
4616               []>;
4617 def TXQ_NUM_MIPMAP_LEVELS
4618   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4619               "txq.num_mipmap_levels.b32 \t$d, [$a];",
4620               []>;
4621 }
4622
4623 def : Pat<(int_nvvm_txq_channel_order Int64Regs:$a),
4624           (TXQ_CHANNEL_ORDER Int64Regs:$a)>;
4625 def : Pat<(int_nvvm_txq_channel_data_type Int64Regs:$a),
4626           (TXQ_CHANNEL_DATA_TYPE Int64Regs:$a)>;
4627 def : Pat<(int_nvvm_txq_width Int64Regs:$a),
4628           (TXQ_WIDTH Int64Regs:$a)>;
4629 def : Pat<(int_nvvm_txq_height Int64Regs:$a),
4630           (TXQ_HEIGHT Int64Regs:$a)>;
4631 def : Pat<(int_nvvm_txq_depth Int64Regs:$a),
4632           (TXQ_DEPTH Int64Regs:$a)>;
4633 def : Pat<(int_nvvm_txq_array_size Int64Regs:$a),
4634           (TXQ_ARRAY_SIZE Int64Regs:$a)>;
4635 def : Pat<(int_nvvm_txq_num_samples Int64Regs:$a),
4636           (TXQ_NUM_SAMPLES Int64Regs:$a)>;
4637 def : Pat<(int_nvvm_txq_num_mipmap_levels Int64Regs:$a),
4638           (TXQ_NUM_MIPMAP_LEVELS Int64Regs:$a)>;
4639
4640
4641 //-----------------------------------
4642 // Surface Query Intrinsics
4643 //-----------------------------------
4644
4645 let IsSurfTexQuery = 1 in {
4646 def SUQ_CHANNEL_ORDER
4647   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4648               "suq.channel_order.b32 \t$d, [$a];",
4649               []>;
4650 def SUQ_CHANNEL_DATA_TYPE
4651   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4652               "suq.channel_data_type.b32 \t$d, [$a];",
4653               []>;
4654 def SUQ_WIDTH
4655   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4656               "suq.width.b32 \t$d, [$a];",
4657               []>;
4658 def SUQ_HEIGHT
4659   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4660               "suq.height.b32 \t$d, [$a];",
4661               []>;
4662 def SUQ_DEPTH
4663   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4664               "suq.depth.b32 \t$d, [$a];",
4665               []>;
4666 def SUQ_ARRAY_SIZE
4667   : NVPTXInst<(outs Int32Regs:$d), (ins Int64Regs:$a),
4668               "suq.array_size.b32 \t$d, [$a];",
4669               []>;
4670 }
4671
4672 def : Pat<(int_nvvm_suq_channel_order Int64Regs:$a),
4673           (SUQ_CHANNEL_ORDER Int64Regs:$a)>;
4674 def : Pat<(int_nvvm_suq_channel_data_type Int64Regs:$a),
4675           (SUQ_CHANNEL_DATA_TYPE Int64Regs:$a)>;
4676 def : Pat<(int_nvvm_suq_width Int64Regs:$a),
4677           (SUQ_WIDTH Int64Regs:$a)>;
4678 def : Pat<(int_nvvm_suq_height Int64Regs:$a),
4679           (SUQ_HEIGHT Int64Regs:$a)>;
4680 def : Pat<(int_nvvm_suq_depth Int64Regs:$a),
4681           (SUQ_DEPTH Int64Regs:$a)>;
4682 def : Pat<(int_nvvm_suq_array_size Int64Regs:$a),
4683           (SUQ_ARRAY_SIZE Int64Regs:$a)>;
4684
4685
4686 //===- Handle Query -------------------------------------------------------===//
4687
4688 // TODO: These intrinsics are not yet finalized, pending PTX ISA design work
4689 def ISTYPEP_SAMPLER
4690   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
4691               "istypep.samplerref \t$d, $a;",
4692               [(set Int1Regs:$d, (int_nvvm_istypep_sampler Int64Regs:$a))]>;
4693 def ISTYPEP_SURFACE
4694   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
4695               "istypep.surfref \t$d, $a;",
4696               [(set Int1Regs:$d, (int_nvvm_istypep_surface Int64Regs:$a))]>;
4697 def ISTYPEP_TEXTURE
4698   : NVPTXInst<(outs Int1Regs:$d), (ins Int64Regs:$a),
4699               "istypep.texref \t$d, $a;",
4700               [(set Int1Regs:$d, (int_nvvm_istypep_texture Int64Regs:$a))]>;
4701
4702 //===- Surface Stores -----------------------------------------------------===//
4703
4704 let IsSust = 1 in {
4705 // Unformatted
4706 // .clamp variant
4707 def SUST_B_1D_B8_CLAMP
4708   : NVPTXInst<(outs),
4709               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
4710               "sust.b.1d.b8.clamp \t[$s, \\{$x\\}], \\{$r\\};",
4711               []>;
4712 def SUST_B_1D_B16_CLAMP
4713   : NVPTXInst<(outs),
4714               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
4715               "sust.b.1d.b16.clamp \t[$s, \\{$x\\}], \\{$r\\};",
4716               []>;
4717 def SUST_B_1D_B32_CLAMP
4718   : NVPTXInst<(outs),
4719               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
4720               "sust.b.1d.b32.clamp \t[$s, \\{$x\\}], \\{$r\\};",
4721               []>;
4722 def SUST_B_1D_B64_CLAMP
4723   : NVPTXInst<(outs),
4724               (ins Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
4725               "sust.b.1d.b64.clamp \t[$s, \\{$x\\}], \\{$r\\};",
4726               []>;
4727 def SUST_B_1D_V2B8_CLAMP
4728   : NVPTXInst<(outs),
4729               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
4730               "sust.b.1d.v2.b8.clamp \t[$s, \\{$x\\}], \\{$r, $g\\};",
4731               []>;
4732 def SUST_B_1D_V2B16_CLAMP
4733   : NVPTXInst<(outs),
4734               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
4735               "sust.b.1d.v2.b16.clamp \t[$s, \\{$x\\}], \\{$r, $g\\};",
4736               []>;
4737 def SUST_B_1D_V2B32_CLAMP
4738   : NVPTXInst<(outs),
4739               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
4740               "sust.b.1d.v2.b32.clamp \t[$s, \\{$x\\}], \\{$r, $g\\};",
4741               []>;
4742 def SUST_B_1D_V2B64_CLAMP
4743   : NVPTXInst<(outs),
4744               (ins Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
4745               "sust.b.1d.v2.b64.clamp \t[$s, \\{$x\\}], \\{$r, $g\\};",
4746               []>;
4747 def SUST_B_1D_V4B8_CLAMP
4748   : NVPTXInst<(outs),
4749               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
4750                    Int16Regs:$b, Int16Regs:$a),
4751               "sust.b.1d.v4.b8.clamp \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
4752               []>;
4753 def SUST_B_1D_V4B16_CLAMP
4754   : NVPTXInst<(outs),
4755               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
4756                    Int16Regs:$b, Int16Regs:$a),
4757               "sust.b.1d.v4.b16.clamp \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
4758               []>;
4759 def SUST_B_1D_V4B32_CLAMP
4760   : NVPTXInst<(outs),
4761               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g,
4762                    Int32Regs:$b, Int32Regs:$a),
4763               "sust.b.1d.v4.b32.clamp \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
4764               []>;
4765
4766
4767 def SUST_B_1D_ARRAY_B8_CLAMP
4768   : NVPTXInst<(outs),
4769               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
4770               "sust.b.a1d.b8.clamp \t[$s, \\{$idx, $x\\}], \\{$r\\};",
4771               []>;
4772 def SUST_B_1D_ARRAY_B16_CLAMP
4773   : NVPTXInst<(outs),
4774               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
4775               "sust.b.a1d.b16.clamp \t[$s, \\{$idx, $x\\}], \\{$r\\};",
4776               []>;
4777 def SUST_B_1D_ARRAY_B32_CLAMP
4778   : NVPTXInst<(outs),
4779               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r),
4780               "sust.b.a1d.b32.clamp \t[$s, \\{$idx, $x\\}], \\{$r\\};",
4781               []>;
4782 def SUST_B_1D_ARRAY_B64_CLAMP
4783   : NVPTXInst<(outs),
4784               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int64Regs:$r),
4785               "sust.b.a1d.b64.clamp \t[$s, \\{$idx, $x\\}], \\{$r\\};",
4786               []>;
4787 def SUST_B_1D_ARRAY_V2B8_CLAMP
4788   : NVPTXInst<(outs),
4789               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
4790                    Int16Regs:$g),
4791               "sust.b.a1d.v2.b8.clamp \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
4792               []>;
4793 def SUST_B_1D_ARRAY_V2B16_CLAMP
4794   : NVPTXInst<(outs),
4795               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
4796                    Int16Regs:$g),
4797               "sust.b.a1d.v2.b16.clamp \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
4798               []>;
4799 def SUST_B_1D_ARRAY_V2B32_CLAMP
4800   : NVPTXInst<(outs),
4801               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
4802                    Int32Regs:$g),
4803               "sust.b.a1d.v2.b32.clamp \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
4804               []>;
4805 def SUST_B_1D_ARRAY_V2B64_CLAMP
4806   : NVPTXInst<(outs),
4807               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int64Regs:$r,
4808                    Int64Regs:$g),
4809               "sust.b.a1d.v2.b64.clamp \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
4810               []>;
4811 def SUST_B_1D_ARRAY_V4B8_CLAMP
4812   : NVPTXInst<(outs),
4813               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
4814                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4815               "sust.b.a1d.v4.b8.clamp \t[$s, \\{$idx, $x\\}], "
4816               "\\{$r, $g, $b, $a\\};",
4817               []>;
4818 def SUST_B_1D_ARRAY_V4B16_CLAMP
4819   : NVPTXInst<(outs),
4820               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
4821                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4822              "sust.b.a1d.v4.b16.clamp \t[$s, \\{$idx, $x\\}], "
4823              "\\{$r, $g, $b, $a\\};",
4824               []>;
4825 def SUST_B_1D_ARRAY_V4B32_CLAMP
4826   : NVPTXInst<(outs),
4827               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
4828                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4829              "sust.b.a1d.v4.b32.clamp \t[$s, \\{$idx, $x\\}], "
4830              "\\{$r, $g, $b, $a\\};",
4831               []>;
4832
4833
4834 def SUST_B_2D_B8_CLAMP
4835   : NVPTXInst<(outs),
4836               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
4837               "sust.b.2d.b8.clamp \t[$s, \\{$x, $y\\}], \\{$r\\};",
4838               []>;
4839 def SUST_B_2D_B16_CLAMP
4840   : NVPTXInst<(outs),
4841               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
4842               "sust.b.2d.b16.clamp \t[$s, \\{$x, $y\\}], \\{$r\\};",
4843               []>;
4844 def SUST_B_2D_B32_CLAMP
4845   : NVPTXInst<(outs),
4846               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
4847               "sust.b.2d.b32.clamp \t[$s, \\{$x, $y\\}], \\{$r\\};",
4848               []>;
4849 def SUST_B_2D_B64_CLAMP
4850   : NVPTXInst<(outs),
4851               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
4852               "sust.b.2d.b64.clamp \t[$s, \\{$x, $y\\}], \\{$r\\};",
4853               []>;
4854 def SUST_B_2D_V2B8_CLAMP
4855   : NVPTXInst<(outs),
4856               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
4857                    Int16Regs:$g),
4858               "sust.b.2d.v2.b8.clamp \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
4859               []>;
4860 def SUST_B_2D_V2B16_CLAMP
4861   : NVPTXInst<(outs),
4862               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
4863                    Int16Regs:$g),
4864               "sust.b.2d.v2.b16.clamp \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
4865               []>;
4866 def SUST_B_2D_V2B32_CLAMP
4867   : NVPTXInst<(outs),
4868               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
4869                    Int32Regs:$g),
4870               "sust.b.2d.v2.b32.clamp \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
4871               []>;
4872 def SUST_B_2D_V2B64_CLAMP
4873   : NVPTXInst<(outs),
4874               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
4875                    Int64Regs:$g),
4876               "sust.b.2d.v2.b64.clamp \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
4877               []>;
4878 def SUST_B_2D_V4B8_CLAMP
4879   : NVPTXInst<(outs),
4880               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
4881                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4882               "sust.b.2d.v4.b8.clamp \t[$s, \\{$x, $y\\}], "
4883               "\\{$r, $g, $b, $a\\};",
4884               []>;
4885 def SUST_B_2D_V4B16_CLAMP
4886   : NVPTXInst<(outs),
4887               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
4888                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4889              "sust.b.2d.v4.b16.clamp \t[$s, \\{$x, $y\\}], "
4890              "\\{$r, $g, $b, $a\\};",
4891               []>;
4892 def SUST_B_2D_V4B32_CLAMP
4893   : NVPTXInst<(outs),
4894               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
4895                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4896              "sust.b.2d.v4.b32.clamp \t[$s, \\{$x, $y\\}], "
4897              "\\{$r, $g, $b, $a\\};",
4898               []>;
4899
4900
4901 def SUST_B_2D_ARRAY_B8_CLAMP
4902   : NVPTXInst<(outs),
4903               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4904                    Int16Regs:$r),
4905               "sust.b.a2d.b8.clamp \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
4906               []>;
4907 def SUST_B_2D_ARRAY_B16_CLAMP
4908   : NVPTXInst<(outs),
4909               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4910                    Int16Regs:$r),
4911               "sust.b.a2d.b16.clamp \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
4912               []>;
4913 def SUST_B_2D_ARRAY_B32_CLAMP
4914   : NVPTXInst<(outs),
4915               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4916                    Int32Regs:$r),
4917               "sust.b.a2d.b32.clamp \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
4918               []>;
4919 def SUST_B_2D_ARRAY_B64_CLAMP
4920   : NVPTXInst<(outs),
4921               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4922                    Int64Regs:$r),
4923               "sust.b.a2d.b64.clamp \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
4924               []>;
4925 def SUST_B_2D_ARRAY_V2B8_CLAMP
4926   : NVPTXInst<(outs),
4927               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4928                    Int16Regs:$r, Int16Regs:$g),
4929               "sust.b.a2d.v2.b8.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4930               "\\{$r, $g\\};",
4931               []>;
4932 def SUST_B_2D_ARRAY_V2B16_CLAMP
4933   : NVPTXInst<(outs),
4934               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4935                    Int16Regs:$r, Int16Regs:$g),
4936              "sust.b.a2d.v2.b16.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4937              "\\{$r, $g\\};",
4938               []>;
4939 def SUST_B_2D_ARRAY_V2B32_CLAMP
4940   : NVPTXInst<(outs),
4941               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4942                    Int32Regs:$r, Int32Regs:$g),
4943              "sust.b.a2d.v2.b32.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4944              "\\{$r, $g\\};",
4945               []>;
4946 def SUST_B_2D_ARRAY_V2B64_CLAMP
4947   : NVPTXInst<(outs),
4948               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4949                    Int64Regs:$r, Int64Regs:$g),
4950              "sust.b.a2d.v2.b64.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4951              "\\{$r, $g\\};",
4952               []>;
4953 def SUST_B_2D_ARRAY_V4B8_CLAMP
4954   : NVPTXInst<(outs),
4955               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4956                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4957       "sust.b.a2d.v4.b8.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4958       "\\{$r, $g, $b, $a\\};",
4959               []>;
4960 def SUST_B_2D_ARRAY_V4B16_CLAMP
4961   : NVPTXInst<(outs),
4962               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4963                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
4964      "sust.b.a2d.v4.b16.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4965      "\\{$r, $g, $b, $a\\};",
4966               []>;
4967 def SUST_B_2D_ARRAY_V4B32_CLAMP
4968   : NVPTXInst<(outs),
4969               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
4970                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
4971      "sust.b.a2d.v4.b32.clamp \t[$s, \\{$idx, $x, $y, $y\\}], "
4972      "\\{$r, $g, $b, $a\\};",
4973               []>;
4974
4975
4976 def SUST_B_3D_B8_CLAMP
4977   : NVPTXInst<(outs),
4978               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4979                    Int16Regs:$r),
4980               "sust.b.3d.b8.clamp \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
4981               []>;
4982 def SUST_B_3D_B16_CLAMP
4983   : NVPTXInst<(outs),
4984               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4985                    Int16Regs:$r),
4986               "sust.b.3d.b16.clamp \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
4987               []>;
4988 def SUST_B_3D_B32_CLAMP
4989   : NVPTXInst<(outs),
4990               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4991                    Int32Regs:$r),
4992               "sust.b.3d.b32.clamp \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
4993               []>;
4994 def SUST_B_3D_B64_CLAMP
4995   : NVPTXInst<(outs),
4996               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
4997                    Int64Regs:$r),
4998               "sust.b.3d.b64.clamp \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
4999               []>;
5000 def SUST_B_3D_V2B8_CLAMP
5001   : NVPTXInst<(outs),
5002               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5003                    Int16Regs:$r, Int16Regs:$g),
5004               "sust.b.3d.v2.b8.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
5005               "\\{$r, $g\\};",
5006               []>;
5007 def SUST_B_3D_V2B16_CLAMP
5008   : NVPTXInst<(outs),
5009               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5010                    Int16Regs:$r, Int16Regs:$g),
5011               "sust.b.3d.v2.b16.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
5012               "\\{$r, $g\\};",
5013               []>;
5014 def SUST_B_3D_V2B32_CLAMP
5015   : NVPTXInst<(outs),
5016               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5017                    Int32Regs:$r, Int32Regs:$g),
5018               "sust.b.3d.v2.b32.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
5019               "\\{$r, $g\\};",
5020               []>;
5021 def SUST_B_3D_V2B64_CLAMP
5022   : NVPTXInst<(outs),
5023               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5024                    Int64Regs:$r, Int64Regs:$g),
5025               "sust.b.3d.v2.b64.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
5026               "\\{$r, $g\\};",
5027               []>;
5028 def SUST_B_3D_V4B8_CLAMP
5029   : NVPTXInst<(outs),
5030               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5031                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5032          "sust.b.3d.v4.b8.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
5033          "\\{$r, $g, $b, $a\\};",
5034               []>;
5035 def SUST_B_3D_V4B16_CLAMP
5036   : NVPTXInst<(outs),
5037               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5038                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5039         "sust.b.3d.v4.b16.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
5040         "\\{$r, $g, $b, $a\\};",
5041               []>;
5042 def SUST_B_3D_V4B32_CLAMP
5043   : NVPTXInst<(outs),
5044               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5045                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5046         "sust.b.3d.v4.b32.clamp \t[$s, \\{$x, $y, $z, $z\\}], "
5047         "\\{$r, $g, $b, $a\\};",
5048               []>;
5049
5050
5051 // .trap variant
5052 def SUST_B_1D_B8_TRAP
5053   : NVPTXInst<(outs),
5054               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
5055               "sust.b.1d.b8.trap \t[$s, \\{$x\\}], \\{$r\\};",
5056               []>;
5057 def SUST_B_1D_B16_TRAP
5058   : NVPTXInst<(outs),
5059               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
5060               "sust.b.1d.b16.trap \t[$s, \\{$x\\}], \\{$r\\};",
5061               []>;
5062 def SUST_B_1D_B32_TRAP
5063   : NVPTXInst<(outs),
5064               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
5065               "sust.b.1d.b32.trap \t[$s, \\{$x\\}], \\{$r\\};",
5066               []>;
5067 def SUST_B_1D_B64_TRAP
5068   : NVPTXInst<(outs),
5069               (ins Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
5070               "sust.b.1d.b64.trap \t[$s, \\{$x\\}], \\{$r\\};",
5071               []>;
5072 def SUST_B_1D_V2B8_TRAP
5073   : NVPTXInst<(outs),
5074               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5075               "sust.b.1d.v2.b8.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
5076               []>;
5077 def SUST_B_1D_V2B16_TRAP
5078   : NVPTXInst<(outs),
5079               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5080               "sust.b.1d.v2.b16.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
5081               []>;
5082 def SUST_B_1D_V2B32_TRAP
5083   : NVPTXInst<(outs),
5084               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
5085               "sust.b.1d.v2.b32.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
5086               []>;
5087 def SUST_B_1D_V2B64_TRAP
5088   : NVPTXInst<(outs),
5089               (ins Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
5090               "sust.b.1d.v2.b64.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
5091               []>;
5092 def SUST_B_1D_V4B8_TRAP
5093   : NVPTXInst<(outs),
5094               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
5095                    Int16Regs:$b, Int16Regs:$a),
5096               "sust.b.1d.v4.b8.trap \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5097               []>;
5098 def SUST_B_1D_V4B16_TRAP
5099   : NVPTXInst<(outs),
5100               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
5101                    Int16Regs:$b, Int16Regs:$a),
5102               "sust.b.1d.v4.b16.trap \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5103               []>;
5104 def SUST_B_1D_V4B32_TRAP
5105   : NVPTXInst<(outs),
5106               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g,
5107                    Int32Regs:$b, Int32Regs:$a),
5108               "sust.b.1d.v4.b32.trap \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5109               []>;
5110
5111
5112 def SUST_B_1D_ARRAY_B8_TRAP
5113   : NVPTXInst<(outs),
5114               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
5115               "sust.b.a1d.b8.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5116               []>;
5117 def SUST_B_1D_ARRAY_B16_TRAP
5118   : NVPTXInst<(outs),
5119               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
5120               "sust.b.a1d.b16.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5121               []>;
5122 def SUST_B_1D_ARRAY_B32_TRAP
5123   : NVPTXInst<(outs),
5124               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r),
5125               "sust.b.a1d.b32.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5126               []>;
5127 def SUST_B_1D_ARRAY_B64_TRAP
5128   : NVPTXInst<(outs),
5129               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int64Regs:$r),
5130               "sust.b.a1d.b64.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5131               []>;
5132 def SUST_B_1D_ARRAY_V2B8_TRAP
5133   : NVPTXInst<(outs),
5134               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5135                    Int16Regs:$g),
5136               "sust.b.a1d.v2.b8.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5137               []>;
5138 def SUST_B_1D_ARRAY_V2B16_TRAP
5139   : NVPTXInst<(outs),
5140               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5141                    Int16Regs:$g),
5142               "sust.b.a1d.v2.b16.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5143               []>;
5144 def SUST_B_1D_ARRAY_V2B32_TRAP
5145   : NVPTXInst<(outs),
5146               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
5147                    Int32Regs:$g),
5148               "sust.b.a1d.v2.b32.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5149               []>;
5150 def SUST_B_1D_ARRAY_V2B64_TRAP
5151   : NVPTXInst<(outs),
5152               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int64Regs:$r,
5153                    Int64Regs:$g),
5154               "sust.b.a1d.v2.b64.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5155               []>;
5156 def SUST_B_1D_ARRAY_V4B8_TRAP
5157   : NVPTXInst<(outs),
5158               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5159                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5160               "sust.b.a1d.v4.b8.trap \t[$s, \\{$idx, $x\\}], "
5161               "\\{$r, $g, $b, $a\\};",
5162               []>;
5163 def SUST_B_1D_ARRAY_V4B16_TRAP
5164   : NVPTXInst<(outs),
5165               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5166                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5167              "sust.b.a1d.v4.b16.trap \t[$s, \\{$idx, $x\\}], "
5168              "\\{$r, $g, $b, $a\\};",
5169               []>;
5170 def SUST_B_1D_ARRAY_V4B32_TRAP
5171   : NVPTXInst<(outs),
5172               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
5173                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5174              "sust.b.a1d.v4.b32.trap \t[$s, \\{$idx, $x\\}], "
5175              "\\{$r, $g, $b, $a\\};",
5176               []>;
5177
5178
5179 def SUST_B_2D_B8_TRAP
5180   : NVPTXInst<(outs),
5181               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5182               "sust.b.2d.b8.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
5183               []>;
5184 def SUST_B_2D_B16_TRAP
5185   : NVPTXInst<(outs),
5186               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5187               "sust.b.2d.b16.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
5188               []>;
5189 def SUST_B_2D_B32_TRAP
5190   : NVPTXInst<(outs),
5191               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
5192               "sust.b.2d.b32.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
5193               []>;
5194 def SUST_B_2D_B64_TRAP
5195   : NVPTXInst<(outs),
5196               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
5197               "sust.b.2d.b64.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
5198               []>;
5199 def SUST_B_2D_V2B8_TRAP
5200   : NVPTXInst<(outs),
5201               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5202                    Int16Regs:$g),
5203               "sust.b.2d.v2.b8.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5204               []>;
5205 def SUST_B_2D_V2B16_TRAP
5206   : NVPTXInst<(outs),
5207               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5208                    Int16Regs:$g),
5209               "sust.b.2d.v2.b16.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5210               []>;
5211 def SUST_B_2D_V2B32_TRAP
5212   : NVPTXInst<(outs),
5213               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
5214                    Int32Regs:$g),
5215               "sust.b.2d.v2.b32.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5216               []>;
5217 def SUST_B_2D_V2B64_TRAP
5218   : NVPTXInst<(outs),
5219               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
5220                    Int64Regs:$g),
5221               "sust.b.2d.v2.b64.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5222               []>;
5223 def SUST_B_2D_V4B8_TRAP
5224   : NVPTXInst<(outs),
5225               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5226                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5227               "sust.b.2d.v4.b8.trap \t[$s, \\{$x, $y\\}], "
5228               "\\{$r, $g, $b, $a\\};",
5229               []>;
5230 def SUST_B_2D_V4B16_TRAP
5231   : NVPTXInst<(outs),
5232               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5233                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5234              "sust.b.2d.v4.b16.trap \t[$s, \\{$x, $y\\}], "
5235              "\\{$r, $g, $b, $a\\};",
5236               []>;
5237 def SUST_B_2D_V4B32_TRAP
5238   : NVPTXInst<(outs),
5239               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
5240                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5241              "sust.b.2d.v4.b32.trap \t[$s, \\{$x, $y\\}], "
5242              "\\{$r, $g, $b, $a\\};",
5243               []>;
5244
5245
5246 def SUST_B_2D_ARRAY_B8_TRAP
5247   : NVPTXInst<(outs),
5248               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5249                    Int16Regs:$r),
5250               "sust.b.a2d.b8.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5251               []>;
5252 def SUST_B_2D_ARRAY_B16_TRAP
5253   : NVPTXInst<(outs),
5254               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5255                    Int16Regs:$r),
5256               "sust.b.a2d.b16.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5257               []>;
5258 def SUST_B_2D_ARRAY_B32_TRAP
5259   : NVPTXInst<(outs),
5260               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5261                    Int32Regs:$r),
5262               "sust.b.a2d.b32.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5263               []>;
5264 def SUST_B_2D_ARRAY_B64_TRAP
5265   : NVPTXInst<(outs),
5266               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5267                    Int64Regs:$r),
5268               "sust.b.a2d.b64.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5269               []>;
5270 def SUST_B_2D_ARRAY_V2B8_TRAP
5271   : NVPTXInst<(outs),
5272               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5273                    Int16Regs:$r, Int16Regs:$g),
5274               "sust.b.a2d.v2.b8.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5275               "\\{$r, $g\\};",
5276               []>;
5277 def SUST_B_2D_ARRAY_V2B16_TRAP
5278   : NVPTXInst<(outs),
5279               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5280                    Int16Regs:$r, Int16Regs:$g),
5281              "sust.b.a2d.v2.b16.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5282              "\\{$r, $g\\};",
5283               []>;
5284 def SUST_B_2D_ARRAY_V2B32_TRAP
5285   : NVPTXInst<(outs),
5286               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5287                    Int32Regs:$r, Int32Regs:$g),
5288              "sust.b.a2d.v2.b32.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5289              "\\{$r, $g\\};",
5290               []>;
5291 def SUST_B_2D_ARRAY_V2B64_TRAP
5292   : NVPTXInst<(outs),
5293               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5294                    Int64Regs:$r, Int64Regs:$g),
5295              "sust.b.a2d.v2.b64.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5296              "\\{$r, $g\\};",
5297               []>;
5298 def SUST_B_2D_ARRAY_V4B8_TRAP
5299   : NVPTXInst<(outs),
5300               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5301                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5302       "sust.b.a2d.v4.b8.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5303       "\\{$r, $g, $b, $a\\};",
5304               []>;
5305 def SUST_B_2D_ARRAY_V4B16_TRAP
5306   : NVPTXInst<(outs),
5307               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5308                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5309      "sust.b.a2d.v4.b16.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5310      "\\{$r, $g, $b, $a\\};",
5311               []>;
5312 def SUST_B_2D_ARRAY_V4B32_TRAP
5313   : NVPTXInst<(outs),
5314               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5315                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5316      "sust.b.a2d.v4.b32.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5317      "\\{$r, $g, $b, $a\\};",
5318               []>;
5319
5320
5321 def SUST_B_3D_B8_TRAP
5322   : NVPTXInst<(outs),
5323               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5324                    Int16Regs:$r),
5325               "sust.b.3d.b8.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5326               []>;
5327 def SUST_B_3D_B16_TRAP
5328   : NVPTXInst<(outs),
5329               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5330                    Int16Regs:$r),
5331               "sust.b.3d.b16.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5332               []>;
5333 def SUST_B_3D_B32_TRAP
5334   : NVPTXInst<(outs),
5335               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5336                    Int32Regs:$r),
5337               "sust.b.3d.b32.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5338               []>;
5339 def SUST_B_3D_B64_TRAP
5340   : NVPTXInst<(outs),
5341               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5342                    Int64Regs:$r),
5343               "sust.b.3d.b64.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5344               []>;
5345 def SUST_B_3D_V2B8_TRAP
5346   : NVPTXInst<(outs),
5347               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5348                    Int16Regs:$r, Int16Regs:$g),
5349               "sust.b.3d.v2.b8.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5350               "\\{$r, $g\\};",
5351               []>;
5352 def SUST_B_3D_V2B16_TRAP
5353   : NVPTXInst<(outs),
5354               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5355                    Int16Regs:$r, Int16Regs:$g),
5356               "sust.b.3d.v2.b16.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5357               "\\{$r, $g\\};",
5358               []>;
5359 def SUST_B_3D_V2B32_TRAP
5360   : NVPTXInst<(outs),
5361               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5362                    Int32Regs:$r, Int32Regs:$g),
5363               "sust.b.3d.v2.b32.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5364               "\\{$r, $g\\};",
5365               []>;
5366 def SUST_B_3D_V2B64_TRAP
5367   : NVPTXInst<(outs),
5368               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5369                    Int64Regs:$r, Int64Regs:$g),
5370               "sust.b.3d.v2.b64.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5371               "\\{$r, $g\\};",
5372               []>;
5373 def SUST_B_3D_V4B8_TRAP
5374   : NVPTXInst<(outs),
5375               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5376                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5377          "sust.b.3d.v4.b8.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5378          "\\{$r, $g, $b, $a\\};",
5379               []>;
5380 def SUST_B_3D_V4B16_TRAP
5381   : NVPTXInst<(outs),
5382               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5383                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5384         "sust.b.3d.v4.b16.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5385         "\\{$r, $g, $b, $a\\};",
5386               []>;
5387 def SUST_B_3D_V4B32_TRAP
5388   : NVPTXInst<(outs),
5389               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5390                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5391         "sust.b.3d.v4.b32.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5392         "\\{$r, $g, $b, $a\\};",
5393               []>;
5394
5395
5396 // .zero variant
5397 def SUST_B_1D_B8_ZERO
5398   : NVPTXInst<(outs),
5399               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
5400               "sust.b.1d.b8.zero \t[$s, \\{$x\\}], \\{$r\\};",
5401               []>;
5402 def SUST_B_1D_B16_ZERO
5403   : NVPTXInst<(outs),
5404               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
5405               "sust.b.1d.b16.zero \t[$s, \\{$x\\}], \\{$r\\};",
5406               []>;
5407 def SUST_B_1D_B32_ZERO
5408   : NVPTXInst<(outs),
5409               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
5410               "sust.b.1d.b32.zero \t[$s, \\{$x\\}], \\{$r\\};",
5411               []>;
5412 def SUST_B_1D_B64_ZERO
5413   : NVPTXInst<(outs),
5414               (ins Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
5415               "sust.b.1d.b64.zero \t[$s, \\{$x\\}], \\{$r\\};",
5416               []>;
5417 def SUST_B_1D_V2B8_ZERO
5418   : NVPTXInst<(outs),
5419               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5420               "sust.b.1d.v2.b8.zero \t[$s, \\{$x\\}], \\{$r, $g\\};",
5421               []>;
5422 def SUST_B_1D_V2B16_ZERO
5423   : NVPTXInst<(outs),
5424               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5425               "sust.b.1d.v2.b16.zero \t[$s, \\{$x\\}], \\{$r, $g\\};",
5426               []>;
5427 def SUST_B_1D_V2B32_ZERO
5428   : NVPTXInst<(outs),
5429               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
5430               "sust.b.1d.v2.b32.zero \t[$s, \\{$x\\}], \\{$r, $g\\};",
5431               []>;
5432 def SUST_B_1D_V2B64_ZERO
5433   : NVPTXInst<(outs),
5434               (ins Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
5435               "sust.b.1d.v2.b64.zero \t[$s, \\{$x\\}], \\{$r, $g\\};",
5436               []>;
5437 def SUST_B_1D_V4B8_ZERO
5438   : NVPTXInst<(outs),
5439               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
5440                    Int16Regs:$b, Int16Regs:$a),
5441               "sust.b.1d.v4.b8.zero \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5442               []>;
5443 def SUST_B_1D_V4B16_ZERO
5444   : NVPTXInst<(outs),
5445               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
5446                    Int16Regs:$b, Int16Regs:$a),
5447               "sust.b.1d.v4.b16.zero \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5448               []>;
5449 def SUST_B_1D_V4B32_ZERO
5450   : NVPTXInst<(outs),
5451               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g,
5452                    Int32Regs:$b, Int32Regs:$a),
5453               "sust.b.1d.v4.b32.zero \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5454               []>;
5455
5456
5457 def SUST_B_1D_ARRAY_B8_ZERO
5458   : NVPTXInst<(outs),
5459               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
5460               "sust.b.a1d.b8.zero \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5461               []>;
5462 def SUST_B_1D_ARRAY_B16_ZERO
5463   : NVPTXInst<(outs),
5464               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
5465               "sust.b.a1d.b16.zero \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5466               []>;
5467 def SUST_B_1D_ARRAY_B32_ZERO
5468   : NVPTXInst<(outs),
5469               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r),
5470               "sust.b.a1d.b32.zero \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5471               []>;
5472 def SUST_B_1D_ARRAY_B64_ZERO
5473   : NVPTXInst<(outs),
5474               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int64Regs:$r),
5475               "sust.b.a1d.b64.zero \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5476               []>;
5477 def SUST_B_1D_ARRAY_V2B8_ZERO
5478   : NVPTXInst<(outs),
5479               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5480                    Int16Regs:$g),
5481               "sust.b.a1d.v2.b8.zero \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5482               []>;
5483 def SUST_B_1D_ARRAY_V2B16_ZERO
5484   : NVPTXInst<(outs),
5485               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5486                    Int16Regs:$g),
5487               "sust.b.a1d.v2.b16.zero \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5488               []>;
5489 def SUST_B_1D_ARRAY_V2B32_ZERO
5490   : NVPTXInst<(outs),
5491               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
5492                    Int32Regs:$g),
5493               "sust.b.a1d.v2.b32.zero \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5494               []>;
5495 def SUST_B_1D_ARRAY_V2B64_ZERO
5496   : NVPTXInst<(outs),
5497               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int64Regs:$r,
5498                    Int64Regs:$g),
5499               "sust.b.a1d.v2.b64.zero \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5500               []>;
5501 def SUST_B_1D_ARRAY_V4B8_ZERO
5502   : NVPTXInst<(outs),
5503               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5504                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5505               "sust.b.a1d.v4.b8.zero \t[$s, \\{$idx, $x\\}], "
5506               "\\{$r, $g, $b, $a\\};",
5507               []>;
5508 def SUST_B_1D_ARRAY_V4B16_ZERO
5509   : NVPTXInst<(outs),
5510               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5511                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5512              "sust.b.a1d.v4.b16.zero \t[$s, \\{$idx, $x\\}], "
5513              "\\{$r, $g, $b, $a\\};",
5514               []>;
5515 def SUST_B_1D_ARRAY_V4B32_ZERO
5516   : NVPTXInst<(outs),
5517               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
5518                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5519              "sust.b.a1d.v4.b32.zero \t[$s, \\{$idx, $x\\}], "
5520              "\\{$r, $g, $b, $a\\};",
5521               []>;
5522
5523
5524 def SUST_B_2D_B8_ZERO
5525   : NVPTXInst<(outs),
5526               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5527               "sust.b.2d.b8.zero \t[$s, \\{$x, $y\\}], \\{$r\\};",
5528               []>;
5529 def SUST_B_2D_B16_ZERO
5530   : NVPTXInst<(outs),
5531               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5532               "sust.b.2d.b16.zero \t[$s, \\{$x, $y\\}], \\{$r\\};",
5533               []>;
5534 def SUST_B_2D_B32_ZERO
5535   : NVPTXInst<(outs),
5536               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
5537               "sust.b.2d.b32.zero \t[$s, \\{$x, $y\\}], \\{$r\\};",
5538               []>;
5539 def SUST_B_2D_B64_ZERO
5540   : NVPTXInst<(outs),
5541               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
5542               "sust.b.2d.b64.zero \t[$s, \\{$x, $y\\}], \\{$r\\};",
5543               []>;
5544 def SUST_B_2D_V2B8_ZERO
5545   : NVPTXInst<(outs),
5546               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5547                    Int16Regs:$g),
5548               "sust.b.2d.v2.b8.zero \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5549               []>;
5550 def SUST_B_2D_V2B16_ZERO
5551   : NVPTXInst<(outs),
5552               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5553                    Int16Regs:$g),
5554               "sust.b.2d.v2.b16.zero \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5555               []>;
5556 def SUST_B_2D_V2B32_ZERO
5557   : NVPTXInst<(outs),
5558               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
5559                    Int32Regs:$g),
5560               "sust.b.2d.v2.b32.zero \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5561               []>;
5562 def SUST_B_2D_V2B64_ZERO
5563   : NVPTXInst<(outs),
5564               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
5565                    Int64Regs:$g),
5566               "sust.b.2d.v2.b64.zero \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5567               []>;
5568 def SUST_B_2D_V4B8_ZERO
5569   : NVPTXInst<(outs),
5570               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5571                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5572               "sust.b.2d.v4.b8.zero \t[$s, \\{$x, $y\\}], "
5573               "\\{$r, $g, $b, $a\\};",
5574               []>;
5575 def SUST_B_2D_V4B16_ZERO
5576   : NVPTXInst<(outs),
5577               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5578                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5579              "sust.b.2d.v4.b16.zero \t[$s, \\{$x, $y\\}], "
5580              "\\{$r, $g, $b, $a\\};",
5581               []>;
5582 def SUST_B_2D_V4B32_ZERO
5583   : NVPTXInst<(outs),
5584               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
5585                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5586              "sust.b.2d.v4.b32.zero \t[$s, \\{$x, $y\\}], "
5587              "\\{$r, $g, $b, $a\\};",
5588               []>;
5589
5590
5591 def SUST_B_2D_ARRAY_B8_ZERO
5592   : NVPTXInst<(outs),
5593               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5594                    Int16Regs:$r),
5595               "sust.b.a2d.b8.zero \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5596               []>;
5597 def SUST_B_2D_ARRAY_B16_ZERO
5598   : NVPTXInst<(outs),
5599               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5600                    Int16Regs:$r),
5601               "sust.b.a2d.b16.zero \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5602               []>;
5603 def SUST_B_2D_ARRAY_B32_ZERO
5604   : NVPTXInst<(outs),
5605               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5606                    Int32Regs:$r),
5607               "sust.b.a2d.b32.zero \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5608               []>;
5609 def SUST_B_2D_ARRAY_B64_ZERO
5610   : NVPTXInst<(outs),
5611               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5612                    Int64Regs:$r),
5613               "sust.b.a2d.b64.zero \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5614               []>;
5615 def SUST_B_2D_ARRAY_V2B8_ZERO
5616   : NVPTXInst<(outs),
5617               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5618                    Int16Regs:$r, Int16Regs:$g),
5619               "sust.b.a2d.v2.b8.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5620               "\\{$r, $g\\};",
5621               []>;
5622 def SUST_B_2D_ARRAY_V2B16_ZERO
5623   : NVPTXInst<(outs),
5624               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5625                    Int16Regs:$r, Int16Regs:$g),
5626              "sust.b.a2d.v2.b16.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5627              "\\{$r, $g\\};",
5628               []>;
5629 def SUST_B_2D_ARRAY_V2B32_ZERO
5630   : NVPTXInst<(outs),
5631               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5632                    Int32Regs:$r, Int32Regs:$g),
5633              "sust.b.a2d.v2.b32.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5634              "\\{$r, $g\\};",
5635               []>;
5636 def SUST_B_2D_ARRAY_V2B64_ZERO
5637   : NVPTXInst<(outs),
5638               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5639                    Int64Regs:$r, Int64Regs:$g),
5640              "sust.b.a2d.v2.b64.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5641              "\\{$r, $g\\};",
5642               []>;
5643 def SUST_B_2D_ARRAY_V4B8_ZERO
5644   : NVPTXInst<(outs),
5645               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5646                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5647       "sust.b.a2d.v4.b8.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5648       "\\{$r, $g, $b, $a\\};",
5649               []>;
5650 def SUST_B_2D_ARRAY_V4B16_ZERO
5651   : NVPTXInst<(outs),
5652               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5653                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5654      "sust.b.a2d.v4.b16.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5655      "\\{$r, $g, $b, $a\\};",
5656               []>;
5657 def SUST_B_2D_ARRAY_V4B32_ZERO
5658   : NVPTXInst<(outs),
5659               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5660                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5661      "sust.b.a2d.v4.b32.zero \t[$s, \\{$idx, $x, $y, $y\\}], "
5662      "\\{$r, $g, $b, $a\\};",
5663               []>;
5664
5665
5666 def SUST_B_3D_B8_ZERO
5667   : NVPTXInst<(outs),
5668               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5669                    Int16Regs:$r),
5670               "sust.b.3d.b8.zero \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5671               []>;
5672 def SUST_B_3D_B16_ZERO
5673   : NVPTXInst<(outs),
5674               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5675                    Int16Regs:$r),
5676               "sust.b.3d.b16.zero \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5677               []>;
5678 def SUST_B_3D_B32_ZERO
5679   : NVPTXInst<(outs),
5680               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5681                    Int32Regs:$r),
5682               "sust.b.3d.b32.zero \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5683               []>;
5684 def SUST_B_3D_B64_ZERO
5685   : NVPTXInst<(outs),
5686               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5687                    Int64Regs:$r),
5688               "sust.b.3d.b64.zero \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5689               []>;
5690 def SUST_B_3D_V2B8_ZERO
5691   : NVPTXInst<(outs),
5692               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5693                    Int16Regs:$r, Int16Regs:$g),
5694               "sust.b.3d.v2.b8.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5695               "\\{$r, $g\\};",
5696               []>;
5697 def SUST_B_3D_V2B16_ZERO
5698   : NVPTXInst<(outs),
5699               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5700                    Int16Regs:$r, Int16Regs:$g),
5701               "sust.b.3d.v2.b16.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5702               "\\{$r, $g\\};",
5703               []>;
5704 def SUST_B_3D_V2B32_ZERO
5705   : NVPTXInst<(outs),
5706               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5707                    Int32Regs:$r, Int32Regs:$g),
5708               "sust.b.3d.v2.b32.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5709               "\\{$r, $g\\};",
5710               []>;
5711 def SUST_B_3D_V2B64_ZERO
5712   : NVPTXInst<(outs),
5713               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5714                    Int64Regs:$r, Int64Regs:$g),
5715               "sust.b.3d.v2.b64.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5716               "\\{$r, $g\\};",
5717               []>;
5718 def SUST_B_3D_V4B8_ZERO
5719   : NVPTXInst<(outs),
5720               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5721                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5722          "sust.b.3d.v4.b8.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5723          "\\{$r, $g, $b, $a\\};",
5724               []>;
5725 def SUST_B_3D_V4B16_ZERO
5726   : NVPTXInst<(outs),
5727               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5728                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5729         "sust.b.3d.v4.b16.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5730         "\\{$r, $g, $b, $a\\};",
5731               []>;
5732 def SUST_B_3D_V4B32_ZERO
5733   : NVPTXInst<(outs),
5734               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5735                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5736         "sust.b.3d.v4.b32.zero \t[$s, \\{$x, $y, $z, $z\\}], "
5737         "\\{$r, $g, $b, $a\\};",
5738               []>;
5739
5740
5741
5742 // Formatted
5743
5744 def SUST_P_1D_B8_TRAP
5745   : NVPTXInst<(outs),
5746               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
5747               "sust.p.1d.b8.trap \t[$s, \\{$x\\}], \\{$r\\};",
5748               []>;
5749 def SUST_P_1D_B16_TRAP
5750   : NVPTXInst<(outs),
5751               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
5752               "sust.p.1d.b16.trap \t[$s, \\{$x\\}], \\{$r\\};",
5753               []>;
5754 def SUST_P_1D_B32_TRAP
5755   : NVPTXInst<(outs),
5756               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
5757               "sust.p.1d.b32.trap \t[$s, \\{$x\\}], \\{$r\\};",
5758               []>;
5759 def SUST_P_1D_V2B8_TRAP
5760   : NVPTXInst<(outs),
5761               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5762               "sust.p.1d.v2.b8.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
5763               []>;
5764 def SUST_P_1D_V2B16_TRAP
5765   : NVPTXInst<(outs),
5766               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
5767               "sust.p.1d.v2.b16.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
5768               []>;
5769 def SUST_P_1D_V2B32_TRAP
5770   : NVPTXInst<(outs),
5771               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
5772               "sust.p.1d.v2.b32.trap \t[$s, \\{$x\\}], \\{$r, $g\\};",
5773               []>;
5774 def SUST_P_1D_V4B8_TRAP
5775   : NVPTXInst<(outs),
5776               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
5777                    Int16Regs:$b, Int16Regs:$a),
5778               "sust.p.1d.v4.b8.trap \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5779               []>;
5780 def SUST_P_1D_V4B16_TRAP
5781   : NVPTXInst<(outs),
5782               (ins Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g,
5783                    Int16Regs:$b, Int16Regs:$a),
5784               "sust.p.1d.v4.b16.trap \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5785               []>;
5786 def SUST_P_1D_V4B32_TRAP
5787   : NVPTXInst<(outs),
5788               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g,
5789                    Int32Regs:$b, Int32Regs:$a),
5790               "sust.p.1d.v4.b32.trap \t[$s, \\{$x\\}], \\{$r, $g, $b, $a\\};",
5791               []>;
5792
5793
5794 def SUST_P_1D_ARRAY_B8_TRAP
5795   : NVPTXInst<(outs),
5796               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
5797               "sust.p.a1d.b8.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5798               []>;
5799 def SUST_P_1D_ARRAY_B16_TRAP
5800   : NVPTXInst<(outs),
5801               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r),
5802               "sust.p.a1d.b16.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5803               []>;
5804 def SUST_P_1D_ARRAY_B32_TRAP
5805   : NVPTXInst<(outs),
5806               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r),
5807               "sust.p.a1d.b32.trap \t[$s, \\{$idx, $x\\}], \\{$r\\};",
5808               []>;
5809 def SUST_P_1D_ARRAY_V2B8_TRAP
5810   : NVPTXInst<(outs),
5811               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5812                    Int16Regs:$g),
5813               "sust.p.a1d.v2.b8.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5814               []>;
5815 def SUST_P_1D_ARRAY_V2B16_TRAP
5816   : NVPTXInst<(outs),
5817               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5818                    Int16Regs:$g),
5819               "sust.p.a1d.v2.b16.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5820               []>;
5821 def SUST_P_1D_ARRAY_V2B32_TRAP
5822   : NVPTXInst<(outs),
5823               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
5824                    Int32Regs:$g),
5825               "sust.p.a1d.v2.b32.trap \t[$s, \\{$idx, $x\\}], \\{$r, $g\\};",
5826               []>;
5827 def SUST_P_1D_ARRAY_V4B8_TRAP
5828   : NVPTXInst<(outs),
5829               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5830                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5831               "sust.p.a1d.v4.b8.trap \t[$s, \\{$idx, $x\\}], "
5832               "\\{$r, $g, $b, $a\\};",
5833               []>;
5834 def SUST_P_1D_ARRAY_V4B16_TRAP
5835   : NVPTXInst<(outs),
5836               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int16Regs:$r,
5837                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5838              "sust.p.a1d.v4.b16.trap \t[$s, \\{$idx, $x\\}], "
5839              "\\{$r, $g, $b, $a\\};",
5840               []>;
5841 def SUST_P_1D_ARRAY_V4B32_TRAP
5842   : NVPTXInst<(outs),
5843               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$r,
5844                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5845              "sust.p.a1d.v4.b32.trap \t[$s, \\{$idx, $x\\}], "
5846              "\\{$r, $g, $b, $a\\};",
5847               []>;
5848
5849
5850 def SUST_P_2D_B8_TRAP
5851   : NVPTXInst<(outs),
5852               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5853               "sust.p.2d.b8.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
5854               []>;
5855 def SUST_P_2D_B16_TRAP
5856   : NVPTXInst<(outs),
5857               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
5858               "sust.p.2d.b16.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
5859               []>;
5860 def SUST_P_2D_B32_TRAP
5861   : NVPTXInst<(outs),
5862               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
5863               "sust.p.2d.b32.trap \t[$s, \\{$x, $y\\}], \\{$r\\};",
5864               []>;
5865 def SUST_P_2D_V2B8_TRAP
5866   : NVPTXInst<(outs),
5867               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5868                    Int16Regs:$g),
5869               "sust.p.2d.v2.b8.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5870               []>;
5871 def SUST_P_2D_V2B16_TRAP
5872   : NVPTXInst<(outs),
5873               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5874                    Int16Regs:$g),
5875               "sust.p.2d.v2.b16.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5876               []>;
5877 def SUST_P_2D_V2B32_TRAP
5878   : NVPTXInst<(outs),
5879               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
5880                    Int32Regs:$g),
5881               "sust.p.2d.v2.b32.trap \t[$s, \\{$x, $y\\}], \\{$r, $g\\};",
5882               []>;
5883 def SUST_P_2D_V4B8_TRAP
5884   : NVPTXInst<(outs),
5885               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5886                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5887               "sust.p.2d.v4.b8.trap \t[$s, \\{$x, $y\\}], "
5888               "\\{$r, $g, $b, $a\\};",
5889               []>;
5890 def SUST_P_2D_V4B16_TRAP
5891   : NVPTXInst<(outs),
5892               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r,
5893                    Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5894              "sust.p.2d.v4.b16.trap \t[$s, \\{$x, $y\\}], "
5895              "\\{$r, $g, $b, $a\\};",
5896               []>;
5897 def SUST_P_2D_V4B32_TRAP
5898   : NVPTXInst<(outs),
5899               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
5900                    Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5901              "sust.p.2d.v4.b32.trap \t[$s, \\{$x, $y\\}], "
5902              "\\{$r, $g, $b, $a\\};",
5903               []>;
5904
5905
5906 def SUST_P_2D_ARRAY_B8_TRAP
5907   : NVPTXInst<(outs),
5908               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5909                    Int16Regs:$r),
5910               "sust.p.a2d.b8.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5911               []>;
5912 def SUST_P_2D_ARRAY_B16_TRAP
5913   : NVPTXInst<(outs),
5914               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5915                    Int16Regs:$r),
5916               "sust.p.a2d.b16.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5917               []>;
5918 def SUST_P_2D_ARRAY_B32_TRAP
5919   : NVPTXInst<(outs),
5920               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5921                    Int32Regs:$r),
5922               "sust.p.a2d.b32.trap \t[$s, \\{$idx, $x, $y, $y\\}], \\{$r\\};",
5923               []>;
5924 def SUST_P_2D_ARRAY_V2B8_TRAP
5925   : NVPTXInst<(outs),
5926               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5927                    Int16Regs:$r, Int16Regs:$g),
5928               "sust.p.a2d.v2.b8.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5929               "\\{$r, $g\\};",
5930               []>;
5931 def SUST_P_2D_ARRAY_V2B16_TRAP
5932   : NVPTXInst<(outs),
5933               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5934                    Int16Regs:$r, Int16Regs:$g),
5935              "sust.p.a2d.v2.b16.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5936              "\\{$r, $g\\};",
5937               []>;
5938 def SUST_P_2D_ARRAY_V2B32_TRAP
5939   : NVPTXInst<(outs),
5940               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5941                    Int32Regs:$r, Int32Regs:$g),
5942              "sust.p.a2d.v2.b32.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5943              "\\{$r, $g\\};",
5944               []>;
5945 def SUST_P_2D_ARRAY_V4B8_TRAP
5946   : NVPTXInst<(outs),
5947               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5948                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5949       "sust.p.a2d.v4.b8.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5950       "\\{$r, $g, $b, $a\\};",
5951               []>;
5952 def SUST_P_2D_ARRAY_V4B16_TRAP
5953   : NVPTXInst<(outs),
5954               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5955                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
5956      "sust.p.a2d.v4.b16.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5957      "\\{$r, $g, $b, $a\\};",
5958               []>;
5959 def SUST_P_2D_ARRAY_V4B32_TRAP
5960   : NVPTXInst<(outs),
5961               (ins Int64Regs:$s, Int32Regs:$idx, Int32Regs:$x, Int32Regs:$y,
5962                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
5963      "sust.p.a2d.v4.b32.trap \t[$s, \\{$idx, $x, $y, $y\\}], "
5964      "\\{$r, $g, $b, $a\\};",
5965               []>;
5966
5967
5968 def SUST_P_3D_B8_TRAP
5969   : NVPTXInst<(outs),
5970               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5971                    Int16Regs:$r),
5972               "sust.p.3d.b8.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5973               []>;
5974 def SUST_P_3D_B16_TRAP
5975   : NVPTXInst<(outs),
5976               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5977                    Int16Regs:$r),
5978               "sust.p.3d.b16.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5979               []>;
5980 def SUST_P_3D_B32_TRAP
5981   : NVPTXInst<(outs),
5982               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5983                    Int32Regs:$r),
5984               "sust.p.3d.b32.trap \t[$s, \\{$x, $y, $z, $z\\}], \\{$r\\};",
5985               []>;
5986 def SUST_P_3D_V2B8_TRAP
5987   : NVPTXInst<(outs),
5988               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5989                    Int16Regs:$r, Int16Regs:$g),
5990               "sust.p.3d.v2.b8.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5991               "\\{$r, $g\\};",
5992               []>;
5993 def SUST_P_3D_V2B16_TRAP
5994   : NVPTXInst<(outs),
5995               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
5996                    Int16Regs:$r, Int16Regs:$g),
5997               "sust.p.3d.v2.b16.trap \t[$s, \\{$x, $y, $z, $z\\}], "
5998               "\\{$r, $g\\};",
5999               []>;
6000 def SUST_P_3D_V2B32_TRAP
6001   : NVPTXInst<(outs),
6002               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6003                    Int32Regs:$r, Int32Regs:$g),
6004               "sust.p.3d.v2.b32.trap \t[$s, \\{$x, $y, $z, $z\\}], "
6005               "\\{$r, $g\\};",
6006               []>;
6007 def SUST_P_3D_V4B8_TRAP
6008   : NVPTXInst<(outs),
6009               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6010                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6011          "sust.p.3d.v4.b8.trap \t[$s, \\{$x, $y, $z, $z\\}], "
6012          "\\{$r, $g, $b, $a\\};",
6013               []>;
6014 def SUST_P_3D_V4B16_TRAP
6015   : NVPTXInst<(outs),
6016               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6017                    Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6018         "sust.p.3d.v4.b16.trap \t[$s, \\{$x, $y, $z, $z\\}], "
6019         "\\{$r, $g, $b, $a\\};",
6020               []>;
6021 def SUST_P_3D_V4B32_TRAP
6022   : NVPTXInst<(outs),
6023               (ins Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6024                    Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6025         "sust.p.3d.v4.b32.trap \t[$s, \\{$x, $y, $z, $z\\}], "
6026         "\\{$r, $g, $b, $a\\};",
6027               []>;
6028 }
6029
6030 // Surface store instruction patterns
6031 // I'm not sure why we can't just include these in the instruction definitions,
6032 // but TableGen complains of type errors :(
6033
6034 // .clamp variant
6035 def : Pat<(int_nvvm_sust_b_1d_i8_clamp
6036            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
6037           (SUST_B_1D_B8_CLAMP Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
6038
6039 def : Pat<(int_nvvm_sust_b_1d_i16_clamp
6040            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
6041           (SUST_B_1D_B16_CLAMP Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
6042
6043 def : Pat<(int_nvvm_sust_b_1d_i32_clamp
6044            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
6045           (SUST_B_1D_B32_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$r)>;
6046
6047 def : Pat<(int_nvvm_sust_b_1d_i64_clamp
6048            Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
6049           (SUST_B_1D_B64_CLAMP Int64Regs:$s, Int32Regs:$x, Int64Regs:$r)>;
6050
6051 def : Pat<(int_nvvm_sust_b_1d_v2i8_clamp
6052            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6053           (SUST_B_1D_V2B8_CLAMP Int64Regs:$s, Int32Regs:$x,
6054            Int16Regs:$r, Int16Regs:$g)>;
6055
6056 def : Pat<(int_nvvm_sust_b_1d_v2i16_clamp
6057            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6058           (SUST_B_1D_V2B16_CLAMP Int64Regs:$s, Int32Regs:$x,
6059            Int16Regs:$r, Int16Regs:$g)>;
6060
6061 def : Pat<(int_nvvm_sust_b_1d_v2i32_clamp
6062            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
6063           (SUST_B_1D_V2B32_CLAMP Int64Regs:$s, Int32Regs:$x,
6064            Int32Regs:$r, Int32Regs:$g)>;
6065
6066 def : Pat<(int_nvvm_sust_b_1d_v2i64_clamp
6067            Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
6068           (SUST_B_1D_V2B64_CLAMP Int64Regs:$s, Int32Regs:$x,
6069            Int64Regs:$r, Int64Regs:$g)>;
6070
6071 def : Pat<(int_nvvm_sust_b_1d_v4i8_clamp
6072            Int64Regs:$s, Int32Regs:$x,
6073            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6074           (SUST_B_1D_V4B8_CLAMP Int64Regs:$s, Int32Regs:$x,
6075            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6076
6077 def : Pat<(int_nvvm_sust_b_1d_v4i16_clamp
6078            Int64Regs:$s, Int32Regs:$x,
6079            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6080           (SUST_B_1D_V4B16_CLAMP Int64Regs:$s, Int32Regs:$x,
6081            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6082
6083 def : Pat<(int_nvvm_sust_b_1d_v4i32_clamp
6084            Int64Regs:$s, Int32Regs:$x,
6085            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6086           (SUST_B_1D_V4B32_CLAMP Int64Regs:$s, Int32Regs:$x,
6087            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6088
6089
6090
6091 def : Pat<(int_nvvm_sust_b_1d_array_i8_clamp
6092            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
6093           (SUST_B_1D_ARRAY_B8_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6094            Int16Regs:$r)>;
6095
6096 def : Pat<(int_nvvm_sust_b_1d_array_i16_clamp
6097            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
6098           (SUST_B_1D_ARRAY_B16_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6099            Int16Regs:$r)>;
6100
6101 def : Pat<(int_nvvm_sust_b_1d_array_i32_clamp
6102            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r),
6103           (SUST_B_1D_ARRAY_B32_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6104            Int32Regs:$r)>;
6105
6106 def : Pat<(int_nvvm_sust_b_1d_array_i64_clamp
6107            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r),
6108           (SUST_B_1D_ARRAY_B64_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6109            Int64Regs:$r)>;
6110
6111 def : Pat<(int_nvvm_sust_b_1d_array_v2i8_clamp
6112           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6113           (SUST_B_1D_ARRAY_V2B8_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6114            Int16Regs:$r, Int16Regs:$g)>;
6115
6116 def : Pat<(int_nvvm_sust_b_1d_array_v2i16_clamp
6117           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6118           (SUST_B_1D_ARRAY_V2B16_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6119            Int16Regs:$r, Int16Regs:$g)>;
6120
6121 def : Pat<(int_nvvm_sust_b_1d_array_v2i32_clamp
6122           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
6123           (SUST_B_1D_ARRAY_V2B32_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6124            Int32Regs:$r, Int32Regs:$g)>;
6125
6126 def : Pat<(int_nvvm_sust_b_1d_array_v2i64_clamp
6127           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
6128           (SUST_B_1D_ARRAY_V2B64_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6129            Int64Regs:$r, Int64Regs:$g)>;
6130
6131 def : Pat<(int_nvvm_sust_b_1d_array_v4i8_clamp
6132            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6133            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6134           (SUST_B_1D_ARRAY_V4B8_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6135            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6136
6137 def : Pat<(int_nvvm_sust_b_1d_array_v4i16_clamp
6138            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6139            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6140           (SUST_B_1D_ARRAY_V4B16_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6141            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6142
6143 def : Pat<(int_nvvm_sust_b_1d_array_v4i32_clamp
6144            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6145            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6146           (SUST_B_1D_ARRAY_V4B32_CLAMP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6147            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6148
6149
6150
6151 def : Pat<(int_nvvm_sust_b_2d_i8_clamp
6152            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6153           (SUST_B_2D_B8_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6154            Int16Regs:$r)>;
6155
6156 def : Pat<(int_nvvm_sust_b_2d_i16_clamp
6157            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6158           (SUST_B_2D_B16_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6159            Int16Regs:$r)>;
6160
6161 def : Pat<(int_nvvm_sust_b_2d_i32_clamp
6162            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
6163           (SUST_B_2D_B32_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6164            Int32Regs:$r)>;
6165
6166 def : Pat<(int_nvvm_sust_b_2d_i64_clamp
6167            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
6168           (SUST_B_2D_B64_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6169            Int64Regs:$r)>;
6170
6171 def : Pat<(int_nvvm_sust_b_2d_v2i8_clamp
6172           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
6173           (SUST_B_2D_V2B8_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6174            Int16Regs:$r, Int16Regs:$g)>;
6175
6176 def : Pat<(int_nvvm_sust_b_2d_v2i16_clamp
6177           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
6178           (SUST_B_2D_V2B16_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6179            Int16Regs:$r, Int16Regs:$g)>;
6180
6181 def : Pat<(int_nvvm_sust_b_2d_v2i32_clamp
6182           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g),
6183           (SUST_B_2D_V2B32_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6184            Int32Regs:$r, Int32Regs:$g)>;
6185
6186 def : Pat<(int_nvvm_sust_b_2d_v2i64_clamp
6187           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g),
6188           (SUST_B_2D_V2B64_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6189            Int64Regs:$r, Int64Regs:$g)>;
6190
6191 def : Pat<(int_nvvm_sust_b_2d_v4i8_clamp
6192            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6193            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6194           (SUST_B_2D_V4B8_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6195            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6196
6197 def : Pat<(int_nvvm_sust_b_2d_v4i16_clamp
6198            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6199            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6200           (SUST_B_2D_V4B16_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6201            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6202
6203 def : Pat<(int_nvvm_sust_b_2d_v4i32_clamp
6204            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6205            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6206           (SUST_B_2D_V4B32_CLAMP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6207            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6208
6209
6210
6211 def : Pat<(int_nvvm_sust_b_2d_array_i8_clamp
6212           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6213           (SUST_B_2D_ARRAY_B8_CLAMP Int64Regs:$s,
6214            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6215            Int16Regs:$r)>;
6216
6217 def : Pat<(int_nvvm_sust_b_2d_array_i16_clamp
6218           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6219           (SUST_B_2D_ARRAY_B16_CLAMP Int64Regs:$s,
6220            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6221            Int16Regs:$r)>;
6222
6223 def : Pat<(int_nvvm_sust_b_2d_array_i32_clamp
6224           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
6225           (SUST_B_2D_ARRAY_B32_CLAMP Int64Regs:$s,
6226            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6227            Int32Regs:$r)>;
6228
6229 def : Pat<(int_nvvm_sust_b_2d_array_i64_clamp
6230           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
6231           (SUST_B_2D_ARRAY_B64_CLAMP Int64Regs:$s,
6232            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6233            Int64Regs:$r)>;
6234
6235 def : Pat<(int_nvvm_sust_b_2d_array_v2i8_clamp
6236            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6237            Int16Regs:$r, Int16Regs:$g),
6238           (SUST_B_2D_ARRAY_V2B8_CLAMP Int64Regs:$s, Int32Regs:$l,
6239            Int32Regs:$x, Int32Regs:$y,
6240            Int16Regs:$r, Int16Regs:$g)>;
6241
6242 def : Pat<(int_nvvm_sust_b_2d_array_v2i16_clamp
6243            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6244            Int16Regs:$r, Int16Regs:$g),
6245           (SUST_B_2D_ARRAY_V2B16_CLAMP Int64Regs:$s, Int32Regs:$l,
6246            Int32Regs:$x, Int32Regs:$y,
6247            Int16Regs:$r, Int16Regs:$g)>;
6248
6249 def : Pat<(int_nvvm_sust_b_2d_array_v2i32_clamp
6250            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
6251            Int32Regs:$g),
6252           (SUST_B_2D_ARRAY_V2B32_CLAMP Int64Regs:$s, Int32Regs:$l,
6253            Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g)>;
6254
6255 def : Pat<(int_nvvm_sust_b_2d_array_v2i64_clamp
6256            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
6257            Int64Regs:$g),
6258           (SUST_B_2D_ARRAY_V2B64_CLAMP Int64Regs:$s, Int32Regs:$l,
6259            Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g)>;
6260
6261 def : Pat<(int_nvvm_sust_b_2d_array_v4i8_clamp
6262            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6263            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6264           (SUST_B_2D_ARRAY_V4B8_CLAMP Int64Regs:$s,
6265            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6266            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6267
6268 def : Pat<(int_nvvm_sust_b_2d_array_v4i16_clamp
6269            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6270            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6271           (SUST_B_2D_ARRAY_V4B16_CLAMP Int64Regs:$s,
6272            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6273            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6274
6275 def : Pat<(int_nvvm_sust_b_2d_array_v4i32_clamp
6276            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6277            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6278           (SUST_B_2D_ARRAY_V4B32_CLAMP Int64Regs:$s, Int32Regs:$l,
6279            Int32Regs:$x, Int32Regs:$y,
6280            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6281
6282
6283
6284 def : Pat<(int_nvvm_sust_b_3d_i8_clamp
6285            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6286            Int16Regs:$r),
6287           (SUST_B_3D_B8_CLAMP Int64Regs:$s,
6288            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6289            Int16Regs:$r)>;
6290
6291 def : Pat<(int_nvvm_sust_b_3d_i16_clamp
6292            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6293            Int16Regs:$r),
6294           (SUST_B_3D_B16_CLAMP Int64Regs:$s,
6295            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6296            Int16Regs:$r)>;
6297
6298 def : Pat<(int_nvvm_sust_b_3d_i32_clamp
6299            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6300            Int32Regs:$r),
6301           (SUST_B_3D_B32_CLAMP Int64Regs:$s,
6302            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6303            Int32Regs:$r)>;
6304
6305 def : Pat<(int_nvvm_sust_b_3d_i64_clamp
6306            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6307            Int64Regs:$r),
6308           (SUST_B_3D_B64_CLAMP Int64Regs:$s,
6309            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6310            Int64Regs:$r)>;
6311
6312 def : Pat<(int_nvvm_sust_b_3d_v2i8_clamp
6313            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6314            Int16Regs:$r, Int16Regs:$g),
6315           (SUST_B_3D_V2B8_CLAMP Int64Regs:$s,
6316            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6317            Int16Regs:$r, Int16Regs:$g)>;
6318
6319 def : Pat<(int_nvvm_sust_b_3d_v2i16_clamp
6320            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6321            Int16Regs:$r, Int16Regs:$g),
6322           (SUST_B_3D_V2B16_CLAMP Int64Regs:$s,
6323            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6324            Int16Regs:$r, Int16Regs:$g)>;
6325
6326 def : Pat<(int_nvvm_sust_b_3d_v2i32_clamp
6327            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6328            Int32Regs:$r, Int32Regs:$g),
6329           (SUST_B_3D_V2B32_CLAMP Int64Regs:$s,
6330            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6331            Int32Regs:$r, Int32Regs:$g)>;
6332
6333 def : Pat<(int_nvvm_sust_b_3d_v2i64_clamp
6334            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6335            Int64Regs:$r, Int64Regs:$g),
6336           (SUST_B_3D_V2B64_CLAMP Int64Regs:$s,
6337            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6338            Int64Regs:$r, Int64Regs:$g)>;
6339
6340 def : Pat<(int_nvvm_sust_b_3d_v4i8_clamp
6341            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6342            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6343           (SUST_B_3D_V4B8_CLAMP Int64Regs:$s,
6344            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6345            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6346
6347 def : Pat<(int_nvvm_sust_b_3d_v4i16_clamp
6348            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6349            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6350           (SUST_B_3D_V4B16_CLAMP Int64Regs:$s,
6351            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6352            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6353
6354 def : Pat<(int_nvvm_sust_b_3d_v4i32_clamp
6355            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6356            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6357           (SUST_B_3D_V4B32_CLAMP Int64Regs:$s,
6358            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6359            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6360
6361
6362 // .trap variant
6363 def : Pat<(int_nvvm_sust_b_1d_i8_trap
6364            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
6365           (SUST_B_1D_B8_TRAP Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
6366
6367 def : Pat<(int_nvvm_sust_b_1d_i16_trap
6368            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
6369           (SUST_B_1D_B16_TRAP Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
6370
6371 def : Pat<(int_nvvm_sust_b_1d_i32_trap
6372            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
6373           (SUST_B_1D_B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$r)>;
6374
6375 def : Pat<(int_nvvm_sust_b_1d_i64_trap
6376            Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
6377           (SUST_B_1D_B64_TRAP Int64Regs:$s, Int32Regs:$x, Int64Regs:$r)>;
6378
6379 def : Pat<(int_nvvm_sust_b_1d_v2i8_trap
6380            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6381           (SUST_B_1D_V2B8_TRAP Int64Regs:$s, Int32Regs:$x,
6382            Int16Regs:$r, Int16Regs:$g)>;
6383
6384 def : Pat<(int_nvvm_sust_b_1d_v2i16_trap
6385            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6386           (SUST_B_1D_V2B16_TRAP Int64Regs:$s, Int32Regs:$x,
6387            Int16Regs:$r, Int16Regs:$g)>;
6388
6389 def : Pat<(int_nvvm_sust_b_1d_v2i32_trap
6390            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
6391           (SUST_B_1D_V2B32_TRAP Int64Regs:$s, Int32Regs:$x,
6392            Int32Regs:$r, Int32Regs:$g)>;
6393
6394 def : Pat<(int_nvvm_sust_b_1d_v2i64_trap
6395            Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
6396           (SUST_B_1D_V2B64_TRAP Int64Regs:$s, Int32Regs:$x,
6397            Int64Regs:$r, Int64Regs:$g)>;
6398
6399 def : Pat<(int_nvvm_sust_b_1d_v4i8_trap
6400            Int64Regs:$s, Int32Regs:$x,
6401            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6402           (SUST_B_1D_V4B8_TRAP Int64Regs:$s, Int32Regs:$x,
6403            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6404
6405 def : Pat<(int_nvvm_sust_b_1d_v4i16_trap
6406            Int64Regs:$s, Int32Regs:$x,
6407            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6408           (SUST_B_1D_V4B16_TRAP Int64Regs:$s, Int32Regs:$x,
6409            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6410
6411 def : Pat<(int_nvvm_sust_b_1d_v4i32_trap
6412            Int64Regs:$s, Int32Regs:$x,
6413            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6414           (SUST_B_1D_V4B32_TRAP Int64Regs:$s, Int32Regs:$x,
6415            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6416
6417
6418
6419 def : Pat<(int_nvvm_sust_b_1d_array_i8_trap
6420            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
6421           (SUST_B_1D_ARRAY_B8_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6422            Int16Regs:$r)>;
6423
6424 def : Pat<(int_nvvm_sust_b_1d_array_i16_trap
6425            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
6426           (SUST_B_1D_ARRAY_B16_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6427            Int16Regs:$r)>;
6428
6429 def : Pat<(int_nvvm_sust_b_1d_array_i32_trap
6430            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r),
6431           (SUST_B_1D_ARRAY_B32_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6432            Int32Regs:$r)>;
6433
6434 def : Pat<(int_nvvm_sust_b_1d_array_i64_trap
6435            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r),
6436           (SUST_B_1D_ARRAY_B64_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6437            Int64Regs:$r)>;
6438
6439 def : Pat<(int_nvvm_sust_b_1d_array_v2i8_trap
6440           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6441           (SUST_B_1D_ARRAY_V2B8_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6442            Int16Regs:$r, Int16Regs:$g)>;
6443
6444 def : Pat<(int_nvvm_sust_b_1d_array_v2i16_trap
6445           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6446           (SUST_B_1D_ARRAY_V2B16_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6447            Int16Regs:$r, Int16Regs:$g)>;
6448
6449 def : Pat<(int_nvvm_sust_b_1d_array_v2i32_trap
6450           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
6451           (SUST_B_1D_ARRAY_V2B32_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6452            Int32Regs:$r, Int32Regs:$g)>;
6453
6454 def : Pat<(int_nvvm_sust_b_1d_array_v2i64_trap
6455           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
6456           (SUST_B_1D_ARRAY_V2B64_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6457            Int64Regs:$r, Int64Regs:$g)>;
6458
6459 def : Pat<(int_nvvm_sust_b_1d_array_v4i8_trap
6460            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6461            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6462           (SUST_B_1D_ARRAY_V4B8_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6463            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6464
6465 def : Pat<(int_nvvm_sust_b_1d_array_v4i16_trap
6466            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6467            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6468           (SUST_B_1D_ARRAY_V4B16_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6469            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6470
6471 def : Pat<(int_nvvm_sust_b_1d_array_v4i32_trap
6472            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6473            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6474           (SUST_B_1D_ARRAY_V4B32_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6475            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6476
6477
6478
6479 def : Pat<(int_nvvm_sust_b_2d_i8_trap
6480            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6481           (SUST_B_2D_B8_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6482            Int16Regs:$r)>;
6483
6484 def : Pat<(int_nvvm_sust_b_2d_i16_trap
6485            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6486           (SUST_B_2D_B16_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6487            Int16Regs:$r)>;
6488
6489 def : Pat<(int_nvvm_sust_b_2d_i32_trap
6490            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
6491           (SUST_B_2D_B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6492            Int32Regs:$r)>;
6493
6494 def : Pat<(int_nvvm_sust_b_2d_i64_trap
6495            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
6496           (SUST_B_2D_B64_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6497            Int64Regs:$r)>;
6498
6499 def : Pat<(int_nvvm_sust_b_2d_v2i8_trap
6500           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
6501           (SUST_B_2D_V2B8_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6502            Int16Regs:$r, Int16Regs:$g)>;
6503
6504 def : Pat<(int_nvvm_sust_b_2d_v2i16_trap
6505           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
6506           (SUST_B_2D_V2B16_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6507            Int16Regs:$r, Int16Regs:$g)>;
6508
6509 def : Pat<(int_nvvm_sust_b_2d_v2i32_trap
6510           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g),
6511           (SUST_B_2D_V2B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6512            Int32Regs:$r, Int32Regs:$g)>;
6513
6514 def : Pat<(int_nvvm_sust_b_2d_v2i64_trap
6515           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g),
6516           (SUST_B_2D_V2B64_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6517            Int64Regs:$r, Int64Regs:$g)>;
6518
6519 def : Pat<(int_nvvm_sust_b_2d_v4i8_trap
6520            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6521            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6522           (SUST_B_2D_V4B8_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6523            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6524
6525 def : Pat<(int_nvvm_sust_b_2d_v4i16_trap
6526            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6527            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6528           (SUST_B_2D_V4B16_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6529            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6530
6531 def : Pat<(int_nvvm_sust_b_2d_v4i32_trap
6532            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6533            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6534           (SUST_B_2D_V4B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6535            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6536
6537
6538
6539 def : Pat<(int_nvvm_sust_b_2d_array_i8_trap
6540           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6541           (SUST_B_2D_ARRAY_B8_TRAP Int64Regs:$s,
6542            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6543            Int16Regs:$r)>;
6544
6545 def : Pat<(int_nvvm_sust_b_2d_array_i16_trap
6546           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6547           (SUST_B_2D_ARRAY_B16_TRAP Int64Regs:$s,
6548            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6549            Int16Regs:$r)>;
6550
6551 def : Pat<(int_nvvm_sust_b_2d_array_i32_trap
6552           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
6553           (SUST_B_2D_ARRAY_B32_TRAP Int64Regs:$s,
6554            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6555            Int32Regs:$r)>;
6556
6557 def : Pat<(int_nvvm_sust_b_2d_array_i64_trap
6558           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
6559           (SUST_B_2D_ARRAY_B64_TRAP Int64Regs:$s,
6560            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6561            Int64Regs:$r)>;
6562
6563 def : Pat<(int_nvvm_sust_b_2d_array_v2i8_trap
6564            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6565            Int16Regs:$r, Int16Regs:$g),
6566           (SUST_B_2D_ARRAY_V2B8_TRAP Int64Regs:$s, Int32Regs:$l,
6567            Int32Regs:$x, Int32Regs:$y,
6568            Int16Regs:$r, Int16Regs:$g)>;
6569
6570 def : Pat<(int_nvvm_sust_b_2d_array_v2i16_trap
6571            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6572            Int16Regs:$r, Int16Regs:$g),
6573           (SUST_B_2D_ARRAY_V2B16_TRAP Int64Regs:$s, Int32Regs:$l,
6574            Int32Regs:$x, Int32Regs:$y,
6575            Int16Regs:$r, Int16Regs:$g)>;
6576
6577 def : Pat<(int_nvvm_sust_b_2d_array_v2i32_trap
6578            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
6579            Int32Regs:$g),
6580           (SUST_B_2D_ARRAY_V2B32_TRAP Int64Regs:$s, Int32Regs:$l,
6581            Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g)>;
6582
6583 def : Pat<(int_nvvm_sust_b_2d_array_v2i64_trap
6584            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
6585            Int64Regs:$g),
6586           (SUST_B_2D_ARRAY_V2B64_TRAP Int64Regs:$s, Int32Regs:$l,
6587            Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g)>;
6588
6589 def : Pat<(int_nvvm_sust_b_2d_array_v4i8_trap
6590            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6591            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6592           (SUST_B_2D_ARRAY_V4B8_TRAP Int64Regs:$s,
6593            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6594            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6595
6596 def : Pat<(int_nvvm_sust_b_2d_array_v4i16_trap
6597            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6598            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6599           (SUST_B_2D_ARRAY_V4B16_TRAP Int64Regs:$s,
6600            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6601            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6602
6603 def : Pat<(int_nvvm_sust_b_2d_array_v4i32_trap
6604            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6605            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6606           (SUST_B_2D_ARRAY_V4B32_TRAP Int64Regs:$s, Int32Regs:$l,
6607            Int32Regs:$x, Int32Regs:$y,
6608            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6609
6610
6611
6612 def : Pat<(int_nvvm_sust_b_3d_i8_trap
6613            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6614            Int16Regs:$r),
6615           (SUST_B_3D_B8_TRAP Int64Regs:$s,
6616            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6617            Int16Regs:$r)>;
6618
6619 def : Pat<(int_nvvm_sust_b_3d_i16_trap
6620            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6621            Int16Regs:$r),
6622           (SUST_B_3D_B16_TRAP Int64Regs:$s,
6623            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6624            Int16Regs:$r)>;
6625
6626 def : Pat<(int_nvvm_sust_b_3d_i32_trap
6627            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6628            Int32Regs:$r),
6629           (SUST_B_3D_B32_TRAP Int64Regs:$s,
6630            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6631            Int32Regs:$r)>;
6632
6633 def : Pat<(int_nvvm_sust_b_3d_i64_trap
6634            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6635            Int64Regs:$r),
6636           (SUST_B_3D_B64_TRAP Int64Regs:$s,
6637            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6638            Int64Regs:$r)>;
6639
6640 def : Pat<(int_nvvm_sust_b_3d_v2i8_trap
6641            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6642            Int16Regs:$r, Int16Regs:$g),
6643           (SUST_B_3D_V2B8_TRAP Int64Regs:$s,
6644            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6645            Int16Regs:$r, Int16Regs:$g)>;
6646
6647 def : Pat<(int_nvvm_sust_b_3d_v2i16_trap
6648            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6649            Int16Regs:$r, Int16Regs:$g),
6650           (SUST_B_3D_V2B16_TRAP Int64Regs:$s,
6651            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6652            Int16Regs:$r, Int16Regs:$g)>;
6653
6654 def : Pat<(int_nvvm_sust_b_3d_v2i32_trap
6655            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6656            Int32Regs:$r, Int32Regs:$g),
6657           (SUST_B_3D_V2B32_TRAP Int64Regs:$s,
6658            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6659            Int32Regs:$r, Int32Regs:$g)>;
6660
6661 def : Pat<(int_nvvm_sust_b_3d_v2i64_trap
6662            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6663            Int64Regs:$r, Int64Regs:$g),
6664           (SUST_B_3D_V2B64_TRAP Int64Regs:$s,
6665            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6666            Int64Regs:$r, Int64Regs:$g)>;
6667
6668 def : Pat<(int_nvvm_sust_b_3d_v4i8_trap
6669            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6670            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6671           (SUST_B_3D_V4B8_TRAP Int64Regs:$s,
6672            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6673            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6674
6675 def : Pat<(int_nvvm_sust_b_3d_v4i16_trap
6676            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6677            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6678           (SUST_B_3D_V4B16_TRAP Int64Regs:$s,
6679            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6680            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6681
6682 def : Pat<(int_nvvm_sust_b_3d_v4i32_trap
6683            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6684            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6685           (SUST_B_3D_V4B32_TRAP Int64Regs:$s,
6686            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6687            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6688
6689
6690 // .zero variant
6691 def : Pat<(int_nvvm_sust_b_1d_i8_zero
6692            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
6693           (SUST_B_1D_B8_ZERO Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
6694
6695 def : Pat<(int_nvvm_sust_b_1d_i16_zero
6696            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
6697           (SUST_B_1D_B16_ZERO Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
6698
6699 def : Pat<(int_nvvm_sust_b_1d_i32_zero
6700            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
6701           (SUST_B_1D_B32_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$r)>;
6702
6703 def : Pat<(int_nvvm_sust_b_1d_i64_zero
6704            Int64Regs:$s, Int32Regs:$x, Int64Regs:$r),
6705           (SUST_B_1D_B64_ZERO Int64Regs:$s, Int32Regs:$x, Int64Regs:$r)>;
6706
6707 def : Pat<(int_nvvm_sust_b_1d_v2i8_zero
6708            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6709           (SUST_B_1D_V2B8_ZERO Int64Regs:$s, Int32Regs:$x,
6710            Int16Regs:$r, Int16Regs:$g)>;
6711
6712 def : Pat<(int_nvvm_sust_b_1d_v2i16_zero
6713            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6714           (SUST_B_1D_V2B16_ZERO Int64Regs:$s, Int32Regs:$x,
6715            Int16Regs:$r, Int16Regs:$g)>;
6716
6717 def : Pat<(int_nvvm_sust_b_1d_v2i32_zero
6718            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
6719           (SUST_B_1D_V2B32_ZERO Int64Regs:$s, Int32Regs:$x,
6720            Int32Regs:$r, Int32Regs:$g)>;
6721
6722 def : Pat<(int_nvvm_sust_b_1d_v2i64_zero
6723            Int64Regs:$s, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
6724           (SUST_B_1D_V2B64_ZERO Int64Regs:$s, Int32Regs:$x,
6725            Int64Regs:$r, Int64Regs:$g)>;
6726
6727 def : Pat<(int_nvvm_sust_b_1d_v4i8_zero
6728            Int64Regs:$s, Int32Regs:$x,
6729            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6730           (SUST_B_1D_V4B8_ZERO Int64Regs:$s, Int32Regs:$x,
6731            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6732
6733 def : Pat<(int_nvvm_sust_b_1d_v4i16_zero
6734            Int64Regs:$s, Int32Regs:$x,
6735            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6736           (SUST_B_1D_V4B16_ZERO Int64Regs:$s, Int32Regs:$x,
6737            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6738
6739 def : Pat<(int_nvvm_sust_b_1d_v4i32_zero
6740            Int64Regs:$s, Int32Regs:$x,
6741            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6742           (SUST_B_1D_V4B32_ZERO Int64Regs:$s, Int32Regs:$x,
6743            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6744
6745
6746
6747 def : Pat<(int_nvvm_sust_b_1d_array_i8_zero
6748            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
6749           (SUST_B_1D_ARRAY_B8_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6750            Int16Regs:$r)>;
6751
6752 def : Pat<(int_nvvm_sust_b_1d_array_i16_zero
6753            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
6754           (SUST_B_1D_ARRAY_B16_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6755            Int16Regs:$r)>;
6756
6757 def : Pat<(int_nvvm_sust_b_1d_array_i32_zero
6758            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r),
6759           (SUST_B_1D_ARRAY_B32_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6760            Int32Regs:$r)>;
6761
6762 def : Pat<(int_nvvm_sust_b_1d_array_i64_zero
6763            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r),
6764           (SUST_B_1D_ARRAY_B64_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6765            Int64Regs:$r)>;
6766
6767 def : Pat<(int_nvvm_sust_b_1d_array_v2i8_zero
6768           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6769           (SUST_B_1D_ARRAY_V2B8_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6770            Int16Regs:$r, Int16Regs:$g)>;
6771
6772 def : Pat<(int_nvvm_sust_b_1d_array_v2i16_zero
6773           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
6774           (SUST_B_1D_ARRAY_V2B16_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6775            Int16Regs:$r, Int16Regs:$g)>;
6776
6777 def : Pat<(int_nvvm_sust_b_1d_array_v2i32_zero
6778           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
6779           (SUST_B_1D_ARRAY_V2B32_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6780            Int32Regs:$r, Int32Regs:$g)>;
6781
6782 def : Pat<(int_nvvm_sust_b_1d_array_v2i64_zero
6783           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int64Regs:$r, Int64Regs:$g),
6784           (SUST_B_1D_ARRAY_V2B64_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6785            Int64Regs:$r, Int64Regs:$g)>;
6786
6787 def : Pat<(int_nvvm_sust_b_1d_array_v4i8_zero
6788            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6789            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6790           (SUST_B_1D_ARRAY_V4B8_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6791            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6792
6793 def : Pat<(int_nvvm_sust_b_1d_array_v4i16_zero
6794            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6795            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6796           (SUST_B_1D_ARRAY_V4B16_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6797            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6798
6799 def : Pat<(int_nvvm_sust_b_1d_array_v4i32_zero
6800            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6801            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6802           (SUST_B_1D_ARRAY_V4B32_ZERO Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
6803            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6804
6805
6806
6807 def : Pat<(int_nvvm_sust_b_2d_i8_zero
6808            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6809           (SUST_B_2D_B8_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6810            Int16Regs:$r)>;
6811
6812 def : Pat<(int_nvvm_sust_b_2d_i16_zero
6813            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6814           (SUST_B_2D_B16_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6815            Int16Regs:$r)>;
6816
6817 def : Pat<(int_nvvm_sust_b_2d_i32_zero
6818            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
6819           (SUST_B_2D_B32_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6820            Int32Regs:$r)>;
6821
6822 def : Pat<(int_nvvm_sust_b_2d_i64_zero
6823            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
6824           (SUST_B_2D_B64_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6825            Int64Regs:$r)>;
6826
6827 def : Pat<(int_nvvm_sust_b_2d_v2i8_zero
6828           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
6829           (SUST_B_2D_V2B8_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6830            Int16Regs:$r, Int16Regs:$g)>;
6831
6832 def : Pat<(int_nvvm_sust_b_2d_v2i16_zero
6833           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
6834           (SUST_B_2D_V2B16_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6835            Int16Regs:$r, Int16Regs:$g)>;
6836
6837 def : Pat<(int_nvvm_sust_b_2d_v2i32_zero
6838           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g),
6839           (SUST_B_2D_V2B32_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6840            Int32Regs:$r, Int32Regs:$g)>;
6841
6842 def : Pat<(int_nvvm_sust_b_2d_v2i64_zero
6843           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g),
6844           (SUST_B_2D_V2B64_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6845            Int64Regs:$r, Int64Regs:$g)>;
6846
6847 def : Pat<(int_nvvm_sust_b_2d_v4i8_zero
6848            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6849            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6850           (SUST_B_2D_V4B8_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6851            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6852
6853 def : Pat<(int_nvvm_sust_b_2d_v4i16_zero
6854            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6855            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6856           (SUST_B_2D_V4B16_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6857            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6858
6859 def : Pat<(int_nvvm_sust_b_2d_v4i32_zero
6860            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6861            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6862           (SUST_B_2D_V4B32_ZERO Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
6863            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6864
6865
6866
6867 def : Pat<(int_nvvm_sust_b_2d_array_i8_zero
6868           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6869           (SUST_B_2D_ARRAY_B8_ZERO Int64Regs:$s,
6870            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6871            Int16Regs:$r)>;
6872
6873 def : Pat<(int_nvvm_sust_b_2d_array_i16_zero
6874           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
6875           (SUST_B_2D_ARRAY_B16_ZERO Int64Regs:$s,
6876            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6877            Int16Regs:$r)>;
6878
6879 def : Pat<(int_nvvm_sust_b_2d_array_i32_zero
6880           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
6881           (SUST_B_2D_ARRAY_B32_ZERO Int64Regs:$s,
6882            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6883            Int32Regs:$r)>;
6884
6885 def : Pat<(int_nvvm_sust_b_2d_array_i64_zero
6886           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r),
6887           (SUST_B_2D_ARRAY_B64_ZERO Int64Regs:$s,
6888            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6889            Int64Regs:$r)>;
6890
6891 def : Pat<(int_nvvm_sust_b_2d_array_v2i8_zero
6892            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6893            Int16Regs:$r, Int16Regs:$g),
6894           (SUST_B_2D_ARRAY_V2B8_ZERO Int64Regs:$s, Int32Regs:$l,
6895            Int32Regs:$x, Int32Regs:$y,
6896            Int16Regs:$r, Int16Regs:$g)>;
6897
6898 def : Pat<(int_nvvm_sust_b_2d_array_v2i16_zero
6899            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6900            Int16Regs:$r, Int16Regs:$g),
6901           (SUST_B_2D_ARRAY_V2B16_ZERO Int64Regs:$s, Int32Regs:$l,
6902            Int32Regs:$x, Int32Regs:$y,
6903            Int16Regs:$r, Int16Regs:$g)>;
6904
6905 def : Pat<(int_nvvm_sust_b_2d_array_v2i32_zero
6906            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
6907            Int32Regs:$g),
6908           (SUST_B_2D_ARRAY_V2B32_ZERO Int64Regs:$s, Int32Regs:$l,
6909            Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g)>;
6910
6911 def : Pat<(int_nvvm_sust_b_2d_array_v2i64_zero
6912            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int64Regs:$r,
6913            Int64Regs:$g),
6914           (SUST_B_2D_ARRAY_V2B64_ZERO Int64Regs:$s, Int32Regs:$l,
6915            Int32Regs:$x, Int32Regs:$y, Int64Regs:$r, Int64Regs:$g)>;
6916
6917 def : Pat<(int_nvvm_sust_b_2d_array_v4i8_zero
6918            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6919            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6920           (SUST_B_2D_ARRAY_V4B8_ZERO Int64Regs:$s,
6921            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6922            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6923
6924 def : Pat<(int_nvvm_sust_b_2d_array_v4i16_zero
6925            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6926            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6927           (SUST_B_2D_ARRAY_V4B16_ZERO Int64Regs:$s,
6928            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6929            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
6930
6931 def : Pat<(int_nvvm_sust_b_2d_array_v4i32_zero
6932            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
6933            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
6934           (SUST_B_2D_ARRAY_V4B32_ZERO Int64Regs:$s, Int32Regs:$l,
6935            Int32Regs:$x, Int32Regs:$y,
6936            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
6937
6938
6939
6940 def : Pat<(int_nvvm_sust_b_3d_i8_zero
6941            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6942            Int16Regs:$r),
6943           (SUST_B_3D_B8_ZERO Int64Regs:$s,
6944            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6945            Int16Regs:$r)>;
6946
6947 def : Pat<(int_nvvm_sust_b_3d_i16_zero
6948            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6949            Int16Regs:$r),
6950           (SUST_B_3D_B16_ZERO Int64Regs:$s,
6951            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6952            Int16Regs:$r)>;
6953
6954 def : Pat<(int_nvvm_sust_b_3d_i32_zero
6955            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6956            Int32Regs:$r),
6957           (SUST_B_3D_B32_ZERO Int64Regs:$s,
6958            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6959            Int32Regs:$r)>;
6960
6961 def : Pat<(int_nvvm_sust_b_3d_i64_zero
6962            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6963            Int64Regs:$r),
6964           (SUST_B_3D_B64_ZERO Int64Regs:$s,
6965            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6966            Int64Regs:$r)>;
6967
6968 def : Pat<(int_nvvm_sust_b_3d_v2i8_zero
6969            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6970            Int16Regs:$r, Int16Regs:$g),
6971           (SUST_B_3D_V2B8_ZERO Int64Regs:$s,
6972            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6973            Int16Regs:$r, Int16Regs:$g)>;
6974
6975 def : Pat<(int_nvvm_sust_b_3d_v2i16_zero
6976            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6977            Int16Regs:$r, Int16Regs:$g),
6978           (SUST_B_3D_V2B16_ZERO Int64Regs:$s,
6979            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6980            Int16Regs:$r, Int16Regs:$g)>;
6981
6982 def : Pat<(int_nvvm_sust_b_3d_v2i32_zero
6983            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6984            Int32Regs:$r, Int32Regs:$g),
6985           (SUST_B_3D_V2B32_ZERO Int64Regs:$s,
6986            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6987            Int32Regs:$r, Int32Regs:$g)>;
6988
6989 def : Pat<(int_nvvm_sust_b_3d_v2i64_zero
6990            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6991            Int64Regs:$r, Int64Regs:$g),
6992           (SUST_B_3D_V2B64_ZERO Int64Regs:$s,
6993            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6994            Int64Regs:$r, Int64Regs:$g)>;
6995
6996 def : Pat<(int_nvvm_sust_b_3d_v4i8_zero
6997            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
6998            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
6999           (SUST_B_3D_V4B8_ZERO Int64Regs:$s,
7000            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7001            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
7002
7003 def : Pat<(int_nvvm_sust_b_3d_v4i16_zero
7004            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7005            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
7006           (SUST_B_3D_V4B16_ZERO Int64Regs:$s,
7007            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7008            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
7009
7010 def : Pat<(int_nvvm_sust_b_3d_v4i32_zero
7011            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7012            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
7013           (SUST_B_3D_V4B32_ZERO Int64Regs:$s,
7014            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7015            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
7016
7017
7018
7019
7020 def : Pat<(int_nvvm_sust_p_1d_i8_trap
7021            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
7022           (SUST_P_1D_B8_TRAP Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
7023
7024 def : Pat<(int_nvvm_sust_p_1d_i16_trap
7025            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r),
7026           (SUST_P_1D_B16_TRAP Int64Regs:$s, Int32Regs:$x, Int16Regs:$r)>;
7027
7028 def : Pat<(int_nvvm_sust_p_1d_i32_trap
7029            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r),
7030           (SUST_P_1D_B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$r)>;
7031
7032 def : Pat<(int_nvvm_sust_p_1d_v2i8_trap
7033            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
7034           (SUST_P_1D_V2B8_TRAP Int64Regs:$s, Int32Regs:$x,
7035            Int16Regs:$r, Int16Regs:$g)>;
7036
7037 def : Pat<(int_nvvm_sust_p_1d_v2i16_trap
7038            Int64Regs:$s, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
7039           (SUST_P_1D_V2B16_TRAP Int64Regs:$s, Int32Regs:$x,
7040            Int16Regs:$r, Int16Regs:$g)>;
7041
7042 def : Pat<(int_nvvm_sust_p_1d_v2i32_trap
7043            Int64Regs:$s, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
7044           (SUST_P_1D_V2B32_TRAP Int64Regs:$s, Int32Regs:$x,
7045            Int32Regs:$r, Int32Regs:$g)>;
7046
7047 def : Pat<(int_nvvm_sust_p_1d_v4i8_trap
7048            Int64Regs:$s, Int32Regs:$x,
7049            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
7050           (SUST_P_1D_V4B8_TRAP Int64Regs:$s, Int32Regs:$x,
7051            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
7052
7053 def : Pat<(int_nvvm_sust_p_1d_v4i16_trap
7054            Int64Regs:$s, Int32Regs:$x,
7055            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
7056           (SUST_P_1D_V4B16_TRAP Int64Regs:$s, Int32Regs:$x,
7057            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
7058
7059 def : Pat<(int_nvvm_sust_p_1d_v4i32_trap
7060            Int64Regs:$s, Int32Regs:$x,
7061            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
7062           (SUST_P_1D_V4B32_TRAP Int64Regs:$s, Int32Regs:$x,
7063            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
7064
7065
7066
7067 def : Pat<(int_nvvm_sust_p_1d_array_i8_trap
7068            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
7069           (SUST_P_1D_ARRAY_B8_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
7070            Int16Regs:$r)>;
7071
7072 def : Pat<(int_nvvm_sust_p_1d_array_i16_trap
7073            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r),
7074           (SUST_P_1D_ARRAY_B16_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
7075            Int16Regs:$r)>;
7076
7077 def : Pat<(int_nvvm_sust_p_1d_array_i32_trap
7078            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r),
7079           (SUST_P_1D_ARRAY_B32_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
7080            Int32Regs:$r)>;
7081
7082 def : Pat<(int_nvvm_sust_p_1d_array_v2i8_trap
7083           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
7084           (SUST_P_1D_ARRAY_V2B8_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
7085            Int16Regs:$r, Int16Regs:$g)>;
7086
7087 def : Pat<(int_nvvm_sust_p_1d_array_v2i16_trap
7088           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int16Regs:$r, Int16Regs:$g),
7089           (SUST_P_1D_ARRAY_V2B16_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
7090            Int16Regs:$r, Int16Regs:$g)>;
7091
7092 def : Pat<(int_nvvm_sust_p_1d_array_v2i32_trap
7093           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$r, Int32Regs:$g),
7094           (SUST_P_1D_ARRAY_V2B32_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
7095            Int32Regs:$r, Int32Regs:$g)>;
7096
7097 def : Pat<(int_nvvm_sust_p_1d_array_v4i8_trap
7098            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
7099            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
7100           (SUST_P_1D_ARRAY_V4B8_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
7101            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
7102
7103 def : Pat<(int_nvvm_sust_p_1d_array_v4i16_trap
7104            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
7105            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
7106           (SUST_P_1D_ARRAY_V4B16_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
7107            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
7108
7109 def : Pat<(int_nvvm_sust_p_1d_array_v4i32_trap
7110            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
7111            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
7112           (SUST_P_1D_ARRAY_V4B32_TRAP Int64Regs:$s, Int32Regs:$l, Int32Regs:$x,
7113            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
7114
7115
7116
7117 def : Pat<(int_nvvm_sust_p_2d_i8_trap
7118            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
7119           (SUST_P_2D_B8_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
7120            Int16Regs:$r)>;
7121
7122 def : Pat<(int_nvvm_sust_p_2d_i16_trap
7123            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
7124           (SUST_P_2D_B16_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
7125            Int16Regs:$r)>;
7126
7127 def : Pat<(int_nvvm_sust_p_2d_i32_trap
7128            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
7129           (SUST_P_2D_B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
7130            Int32Regs:$r)>;
7131
7132 def : Pat<(int_nvvm_sust_p_2d_v2i8_trap
7133           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
7134           (SUST_P_2D_V2B8_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
7135            Int16Regs:$r, Int16Regs:$g)>;
7136
7137 def : Pat<(int_nvvm_sust_p_2d_v2i16_trap
7138           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r, Int16Regs:$g),
7139           (SUST_P_2D_V2B16_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
7140            Int16Regs:$r, Int16Regs:$g)>;
7141
7142 def : Pat<(int_nvvm_sust_p_2d_v2i32_trap
7143           Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g),
7144           (SUST_P_2D_V2B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
7145            Int32Regs:$r, Int32Regs:$g)>;
7146
7147 def : Pat<(int_nvvm_sust_p_2d_v4i8_trap
7148            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
7149            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
7150           (SUST_P_2D_V4B8_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
7151            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
7152
7153 def : Pat<(int_nvvm_sust_p_2d_v4i16_trap
7154            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
7155            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
7156           (SUST_P_2D_V4B16_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
7157            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
7158
7159 def : Pat<(int_nvvm_sust_p_2d_v4i32_trap
7160            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
7161            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
7162           (SUST_P_2D_V4B32_TRAP Int64Regs:$s, Int32Regs:$x, Int32Regs:$y,
7163            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
7164
7165
7166
7167 def : Pat<(int_nvvm_sust_p_2d_array_i8_trap
7168           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
7169           (SUST_P_2D_ARRAY_B8_TRAP Int64Regs:$s,
7170            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
7171            Int16Regs:$r)>;
7172
7173 def : Pat<(int_nvvm_sust_p_2d_array_i16_trap
7174           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int16Regs:$r),
7175           (SUST_P_2D_ARRAY_B16_TRAP Int64Regs:$s,
7176            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
7177            Int16Regs:$r)>;
7178
7179 def : Pat<(int_nvvm_sust_p_2d_array_i32_trap
7180           Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r),
7181           (SUST_P_2D_ARRAY_B32_TRAP Int64Regs:$s,
7182            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
7183            Int32Regs:$r)>;
7184
7185 def : Pat<(int_nvvm_sust_p_2d_array_v2i8_trap
7186            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
7187            Int16Regs:$r, Int16Regs:$g),
7188           (SUST_P_2D_ARRAY_V2B8_TRAP Int64Regs:$s, Int32Regs:$l,
7189            Int32Regs:$x, Int32Regs:$y,
7190            Int16Regs:$r, Int16Regs:$g)>;
7191
7192 def : Pat<(int_nvvm_sust_p_2d_array_v2i16_trap
7193            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
7194            Int16Regs:$r, Int16Regs:$g),
7195           (SUST_P_2D_ARRAY_V2B16_TRAP Int64Regs:$s, Int32Regs:$l,
7196            Int32Regs:$x, Int32Regs:$y,
7197            Int16Regs:$r, Int16Regs:$g)>;
7198
7199 def : Pat<(int_nvvm_sust_p_2d_array_v2i32_trap
7200            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y, Int32Regs:$r,
7201            Int32Regs:$g),
7202           (SUST_P_2D_ARRAY_V2B32_TRAP Int64Regs:$s, Int32Regs:$l,
7203            Int32Regs:$x, Int32Regs:$y, Int32Regs:$r, Int32Regs:$g)>;
7204
7205 def : Pat<(int_nvvm_sust_p_2d_array_v4i8_trap
7206            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
7207            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
7208           (SUST_P_2D_ARRAY_V4B8_TRAP Int64Regs:$s,
7209            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
7210            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
7211
7212 def : Pat<(int_nvvm_sust_p_2d_array_v4i16_trap
7213            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
7214            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
7215           (SUST_P_2D_ARRAY_V4B16_TRAP Int64Regs:$s,
7216            Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
7217            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
7218
7219 def : Pat<(int_nvvm_sust_p_2d_array_v4i32_trap
7220            Int64Regs:$s, Int32Regs:$l, Int32Regs:$x, Int32Regs:$y,
7221            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
7222           (SUST_P_2D_ARRAY_V4B32_TRAP Int64Regs:$s, Int32Regs:$l,
7223            Int32Regs:$x, Int32Regs:$y,
7224            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
7225
7226
7227
7228 def : Pat<(int_nvvm_sust_p_3d_i8_trap
7229            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7230            Int16Regs:$r),
7231           (SUST_P_3D_B8_TRAP Int64Regs:$s,
7232            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7233            Int16Regs:$r)>;
7234
7235 def : Pat<(int_nvvm_sust_p_3d_i16_trap
7236            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7237            Int16Regs:$r),
7238           (SUST_P_3D_B16_TRAP Int64Regs:$s,
7239            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7240            Int16Regs:$r)>;
7241
7242 def : Pat<(int_nvvm_sust_p_3d_i32_trap
7243            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7244            Int32Regs:$r),
7245           (SUST_P_3D_B32_TRAP Int64Regs:$s,
7246            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7247            Int32Regs:$r)>;
7248
7249 def : Pat<(int_nvvm_sust_p_3d_v2i8_trap
7250            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7251            Int16Regs:$r, Int16Regs:$g),
7252           (SUST_P_3D_V2B8_TRAP Int64Regs:$s,
7253            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7254            Int16Regs:$r, Int16Regs:$g)>;
7255
7256 def : Pat<(int_nvvm_sust_p_3d_v2i16_trap
7257            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7258            Int16Regs:$r, Int16Regs:$g),
7259           (SUST_P_3D_V2B16_TRAP Int64Regs:$s,
7260            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7261            Int16Regs:$r, Int16Regs:$g)>;
7262
7263 def : Pat<(int_nvvm_sust_p_3d_v2i32_trap
7264            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7265            Int32Regs:$r, Int32Regs:$g),
7266           (SUST_P_3D_V2B32_TRAP Int64Regs:$s,
7267            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7268            Int32Regs:$r, Int32Regs:$g)>;
7269
7270 def : Pat<(int_nvvm_sust_p_3d_v4i8_trap
7271            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7272            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
7273           (SUST_P_3D_V4B8_TRAP Int64Regs:$s,
7274            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7275            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
7276
7277 def : Pat<(int_nvvm_sust_p_3d_v4i16_trap
7278            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7279            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a),
7280           (SUST_P_3D_V4B16_TRAP Int64Regs:$s,
7281            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7282            Int16Regs:$r, Int16Regs:$g, Int16Regs:$b, Int16Regs:$a)>;
7283
7284 def : Pat<(int_nvvm_sust_p_3d_v4i32_trap
7285            Int64Regs:$s, Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7286            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a),
7287           (SUST_P_3D_V4B32_TRAP Int64Regs:$s,
7288            Int32Regs:$x, Int32Regs:$y, Int32Regs:$z,
7289            Int32Regs:$r, Int32Regs:$g, Int32Regs:$b, Int32Regs:$a)>;
7290
7291 //-----------------------------------
7292 // Read Special Registers
7293 //-----------------------------------
7294
7295 class PTX_READ_SREG_R64<string regname, Intrinsic intop>
7296   : NVPTXInst<(outs Int64Regs:$d), (ins),
7297               !strconcat("mov.u64 \t$d, %", regname, ";"),
7298               [(set Int64Regs:$d, (intop))]>;
7299
7300 class PTX_READ_SREG_R32<string regname, Intrinsic intop>
7301   : NVPTXInst<(outs Int32Regs:$d), (ins),
7302               !strconcat("mov.u32 \t$d, %", regname, ";"),
7303               [(set Int32Regs:$d, (intop))]>;
7304
7305 // TODO Add read vector-version of special registers
7306
7307 def INT_PTX_SREG_TID_X :
7308     PTX_READ_SREG_R32<"tid.x", int_nvvm_read_ptx_sreg_tid_x>;
7309 def INT_PTX_SREG_TID_Y :
7310     PTX_READ_SREG_R32<"tid.y", int_nvvm_read_ptx_sreg_tid_y>;
7311 def INT_PTX_SREG_TID_Z :
7312     PTX_READ_SREG_R32<"tid.z", int_nvvm_read_ptx_sreg_tid_z>;
7313 def INT_PTX_SREG_TID_W :
7314     PTX_READ_SREG_R32<"tid.w", int_nvvm_read_ptx_sreg_tid_w>;
7315
7316 def INT_PTX_SREG_NTID_X :
7317     PTX_READ_SREG_R32<"ntid.x", int_nvvm_read_ptx_sreg_ntid_x>;
7318 def INT_PTX_SREG_NTID_Y :
7319     PTX_READ_SREG_R32<"ntid.y", int_nvvm_read_ptx_sreg_ntid_y>;
7320 def INT_PTX_SREG_NTID_Z :
7321     PTX_READ_SREG_R32<"ntid.z", int_nvvm_read_ptx_sreg_ntid_z>;
7322 def INT_PTX_SREG_NTID_W :
7323     PTX_READ_SREG_R32<"ntid.w", int_nvvm_read_ptx_sreg_ntid_w>;
7324
7325 def INT_PTX_SREG_LANEID :
7326     PTX_READ_SREG_R32<"laneid", int_nvvm_read_ptx_sreg_laneid>;
7327 def INT_PTX_SREG_WARPID :
7328     PTX_READ_SREG_R32<"warpid", int_nvvm_read_ptx_sreg_warpid>;
7329 def INT_PTX_SREG_NWARPID :
7330     PTX_READ_SREG_R32<"nwarpid", int_nvvm_read_ptx_sreg_nwarpid>;
7331
7332 def INT_PTX_SREG_CTAID_X :
7333     PTX_READ_SREG_R32<"ctaid.x", int_nvvm_read_ptx_sreg_ctaid_x>;
7334 def INT_PTX_SREG_CTAID_Y :
7335     PTX_READ_SREG_R32<"ctaid.y", int_nvvm_read_ptx_sreg_ctaid_y>;
7336 def INT_PTX_SREG_CTAID_Z :
7337     PTX_READ_SREG_R32<"ctaid.z", int_nvvm_read_ptx_sreg_ctaid_z>;
7338 def INT_PTX_SREG_CTAID_W :
7339     PTX_READ_SREG_R32<"ctaid.w", int_nvvm_read_ptx_sreg_ctaid_w>;
7340
7341 def INT_PTX_SREG_NCTAID_X :
7342     PTX_READ_SREG_R32<"nctaid.x", int_nvvm_read_ptx_sreg_nctaid_x>;
7343 def INT_PTX_SREG_NCTAID_Y :
7344     PTX_READ_SREG_R32<"nctaid.y", int_nvvm_read_ptx_sreg_nctaid_y>;
7345 def INT_PTX_SREG_NCTAID_Z :
7346     PTX_READ_SREG_R32<"nctaid.z", int_nvvm_read_ptx_sreg_nctaid_z>;
7347 def INT_PTX_SREG_NCTAID_W :
7348     PTX_READ_SREG_R32<"nctaid.w", int_nvvm_read_ptx_sreg_nctaid_w>;
7349
7350 def INT_PTX_SREG_SMID :
7351     PTX_READ_SREG_R32<"smid", int_nvvm_read_ptx_sreg_smid>;
7352 def INT_PTX_SREG_NSMID :
7353     PTX_READ_SREG_R32<"nsmid", int_nvvm_read_ptx_sreg_nsmid>;
7354 def INT_PTX_SREG_GRIDID :
7355     PTX_READ_SREG_R32<"gridid", int_nvvm_read_ptx_sreg_gridid>;
7356
7357 def INT_PTX_SREG_LANEMASK_EQ :
7358     PTX_READ_SREG_R32<"lanemask_eq", int_nvvm_read_ptx_sreg_lanemask_eq>;
7359 def INT_PTX_SREG_LANEMASK_LE :
7360     PTX_READ_SREG_R32<"lanemask_le", int_nvvm_read_ptx_sreg_lanemask_le>;
7361 def INT_PTX_SREG_LANEMASK_LT :
7362     PTX_READ_SREG_R32<"lanemask_lt", int_nvvm_read_ptx_sreg_lanemask_lt>;
7363 def INT_PTX_SREG_LANEMASK_GE :
7364     PTX_READ_SREG_R32<"lanemask_ge", int_nvvm_read_ptx_sreg_lanemask_ge>;
7365 def INT_PTX_SREG_LANEMASK_GT :
7366     PTX_READ_SREG_R32<"lanemask_gt", int_nvvm_read_ptx_sreg_lanemask_gt>;
7367
7368 def INT_PTX_SREG_CLOCK :
7369     PTX_READ_SREG_R32<"clock", int_nvvm_read_ptx_sreg_clock>;
7370 def INT_PTX_SREG_CLOCK64 :
7371     PTX_READ_SREG_R64<"clock64", int_nvvm_read_ptx_sreg_clock64>;
7372
7373 def INT_PTX_SREG_PM0 : PTX_READ_SREG_R32<"pm0", int_nvvm_read_ptx_sreg_pm0>;
7374 def INT_PTX_SREG_PM1 : PTX_READ_SREG_R32<"pm1", int_nvvm_read_ptx_sreg_pm1>;
7375 def INT_PTX_SREG_PM2 : PTX_READ_SREG_R32<"pm2", int_nvvm_read_ptx_sreg_pm2>;
7376 def INT_PTX_SREG_PM3 : PTX_READ_SREG_R32<"pm3", int_nvvm_read_ptx_sreg_pm3>;
7377
7378 // TODO: It would be nice to use PTX_READ_SREG here, but it doesn't
7379 // handle the constant.
7380 def INT_PTX_SREG_WARPSIZE :
7381     NVPTXInst<(outs Int32Regs:$dst), (ins), "mov.u32 \t$dst, WARP_SZ;",
7382               [(set Int32Regs:$dst, (int_nvvm_read_ptx_sreg_warpsize))]>;
7383
7384 //
7385 // wmma.load.[a|b|c].sync.[row|col].m16n16k16[|.global|.shared].[f16|f32]
7386 //
7387
7388 class EmptyNVPTXInst : NVPTXInst<(outs), (ins), "?", []>;
7389
7390 class WMMA_LOAD_GALSTOS<string Geometry, string Abc, string Layout,
7391                         string Space, string Type, NVPTXRegClass regclass,
7392                         DAGOperand SrcOp, bit WithStride>
7393   : EmptyNVPTXInst,
7394     Requires<[!if(!eq(Geometry, "m16n16k16"),
7395                   hasPTX60,
7396                   hasPTX61),
7397               hasSM70]> {
7398   // Pattern (created by WMMA_LOAD_INTR_HELPER below) that matches the intrinsic
7399   // for this function.
7400   PatFrag IntrMatcher = !cast<PatFrag>("INT_WMMA_"
7401                                        # Geometry # "_load_"
7402                                        # !subst("c", "c_" # Type, Abc)
7403                                        # "_" # Layout
7404                                        # !subst(".", "_", Space)
7405                                        # !if(WithStride,"_stride", "")
7406                                        # "_Intr");
7407   dag OutsR03 = (outs regclass:$r0, regclass:$r1, regclass:$r2, regclass:$r3);
7408   dag OutsR47 = (outs regclass:$r4, regclass:$r5, regclass:$r6, regclass:$r7);
7409   dag Outs = !if(!eq(Abc#Type,"cf16"), OutsR03, !con(OutsR03, OutsR47));
7410
7411   dag StrideArg = !if(WithStride, (ins Int32Regs:$ldm), (ins));
7412   dag Ins = !con((ins SrcOp:$src), StrideArg);
7413
7414   // Build a dag pattern that matches the intrinsic call.
7415   // We want a dag that looks like this:
7416   // (set <output args>, (intrinsic <input arguments>)) where input and
7417   // output arguments are named patterns that would match corresponding
7418   // input/output arguments of the instruction.
7419   //
7420   // First we construct (set <output arguments>) from instruction's outs dag by
7421   // replacing dag operator 'outs' with 'set'.
7422   dag PatOuts = !foreach(tmp, Outs, !subst(outs, set, tmp));
7423   // Similarly, construct (intrinsic <input arguments>) sub-dag from
7424   // instruction's input arguments, only now we also need to replace operands
7425   // with patterns that would match them and the operator 'ins' with the
7426   // intrinsic.
7427   dag PatArgs = !foreach(tmp, Ins,
7428                               !subst(imem, ADDRvar,
7429                               !subst(MEMri64, ADDRri64,
7430                               !subst(MEMri, ADDRri,
7431                               !subst(ins, IntrMatcher, tmp)))));
7432   // Finally, consatenate both parts together. !con() requires both dags to have
7433   // the same operator, so we wrap PatArgs in a (set ...) dag.
7434   let Pattern = [!con(PatOuts, (set PatArgs))];
7435   let OutOperandList = Outs;
7436   let InOperandList = Ins;
7437   let AsmString = "wmma.load."
7438                   # Abc
7439                   # ".sync"
7440                   # "." # Layout
7441                   # "." # Geometry
7442                   # Space
7443                   # "." # Type # " \t"
7444                   # !if(!eq(Abc#Type, "cf16"),
7445                         "{{$r0, $r1, $r2, $r3}}",
7446                         "{{$r0, $r1, $r2, $r3, $r4, $r5, $r6, $r7}}")
7447                   # ", [$src]"
7448                   # !if(WithStride, ", $ldm", "")
7449                   # ";";
7450 }
7451
7452 class WMMA_LOAD_INTR_HELPER<string Geometry, string Abc, string Layout,
7453                             string Space, string Type, bit WithStride>
7454                            : PatFrag <(ops),(ops)> {
7455   // Intrinsic that matches this instruction.
7456   Intrinsic Intr = !cast<Intrinsic>("int_nvvm_wmma"
7457                                     # "_" # Geometry # "_load_"
7458                                     # Abc # "_" # Type # "_" # Layout
7459                                     # !if(WithStride,"_stride", ""));
7460   code match_generic = [{
7461    return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_GENERIC);
7462   }];
7463   code match_shared = [{
7464    return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_SHARED);
7465   }];
7466   code match_global = [{
7467    return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_GLOBAL);
7468   }];
7469
7470   let Operands = !if(WithStride, (ops node:$src, node:$ldm), (ops node:$src));
7471   let Fragments = [!foreach(tmp, Operands, !subst(ops, Intr, tmp))];
7472   let PredicateCode = !if(!eq(Space, ".shared"), match_shared,
7473                       !if(!eq(Space, ".global"), match_global, match_generic));
7474 }
7475
7476 multiclass WMMA_LOAD_GALSTS<string Geometry, string Abc, string Layout,
7477                             string Space, string Type, NVPTXRegClass regclass,
7478                             bit WithStride> {
7479   def _avar:  WMMA_LOAD_GALSTOS<Geometry, Abc, Layout, Space, Type, regclass,
7480                                 imem, WithStride>;
7481   def _areg: WMMA_LOAD_GALSTOS<Geometry, Abc, Layout, Space, Type, regclass,
7482                                 Int32Regs, WithStride>;
7483   def _areg64: WMMA_LOAD_GALSTOS<Geometry, Abc, Layout, Space, Type, regclass,
7484                                 Int64Regs, WithStride>;
7485   def _ari: WMMA_LOAD_GALSTOS<Geometry, Abc, Layout, Space, Type, regclass,
7486                                 MEMri, WithStride>;
7487   def _ari64: WMMA_LOAD_GALSTOS<Geometry, Abc, Layout, Space, Type, regclass,
7488                                 MEMri64, WithStride>;
7489 }
7490
7491 multiclass WMMA_LOAD_GALSTSh<string Geometry, string Abc, string Layout,
7492                              string Space, string Type, NVPTXRegClass regclass,
7493                              bit WithStride> {
7494   // Define a PatFrag that matches appropriate intrinsic that loads from the
7495   // given address space.
7496   def _Intr:  WMMA_LOAD_INTR_HELPER<Geometry, Abc, Layout, Space, Type,
7497                                     WithStride>;
7498   defm NAME:  WMMA_LOAD_GALSTS<Geometry, Abc, Layout, Space, Type, regclass,
7499                                WithStride>;
7500 }
7501
7502 multiclass WMMA_LOAD_GALST<string Geometry, string Abc, string Layout,
7503                            string Space, string Type, NVPTXRegClass regclass> {
7504   defm _stride: WMMA_LOAD_GALSTSh<Geometry, Abc, Layout, Space, Type, regclass, 1>;
7505   defm NAME:    WMMA_LOAD_GALSTSh<Geometry, Abc, Layout, Space, Type, regclass, 0>;
7506 }
7507
7508 multiclass WMMA_LOAD_GALT<string Geometry, string Abc, string Layout,
7509                           string Type, NVPTXRegClass regclass> {
7510   defm _global: WMMA_LOAD_GALST<Geometry, Abc, Layout, ".global",
7511                                 Type, regclass>;
7512   defm _shared: WMMA_LOAD_GALST<Geometry, Abc, Layout, ".shared",
7513                                 Type, regclass>;
7514   defm NAME:    WMMA_LOAD_GALST<Geometry, Abc, Layout,        "",
7515                                 Type, regclass>;
7516 }
7517
7518 multiclass WMMA_LOAD_GAT<string Geometry, string Abc,
7519                          string Type, NVPTXRegClass regclass> {
7520   defm _row: WMMA_LOAD_GALT<Geometry, Abc, "row", Type, regclass>;
7521   defm _col: WMMA_LOAD_GALT<Geometry, Abc, "col", Type, regclass>;
7522 }
7523
7524 multiclass WMMA_LOAD_G<string Geometry> {
7525   defm _load_a: WMMA_LOAD_GAT<Geometry, "a", "f16", Float16x2Regs>;
7526   defm _load_b: WMMA_LOAD_GAT<Geometry, "b", "f16", Float16x2Regs>;
7527   defm _load_c_f16: WMMA_LOAD_GAT<Geometry, "c", "f16", Float16x2Regs>;
7528   defm _load_c_f32: WMMA_LOAD_GAT<Geometry, "c", "f32", Float32Regs>;
7529 }
7530
7531 defm INT_WMMA_m32n8k16: WMMA_LOAD_G<"m32n8k16">;
7532 defm INT_WMMA_m16n16k16: WMMA_LOAD_G<"m16n16k16">;
7533 defm INT_WMMA_m8n32k16: WMMA_LOAD_G<"m8n32k16">;
7534
7535 //
7536 // wmma.store.d.sync.[row|col].m16n16k16[|.global|.shared].[f16|f32]
7537 //
7538 class WMMA_STORE_D_GLSTSO<string Geometry, string Layout, string Space,
7539                           string Type, NVPTXRegClass regclass,
7540                           bit WithStride, DAGOperand DstOp>
7541   : EmptyNVPTXInst,
7542     Requires<[!if(!eq(Geometry, "m16n16k16"),
7543                   hasPTX60,
7544                   hasPTX61),
7545               hasSM70]> {
7546   PatFrag IntrMatcher = !cast<PatFrag>("INT_WMMA"
7547                                        # "_" # Geometry # "_store_d"
7548                                        # "_" # Type
7549                                        # "_" # Layout
7550                                        # !subst(".", "_", Space)
7551                                        # !if(WithStride,"_stride", "")
7552                                        # "_Intr");
7553   dag InsR03 = (ins DstOp:$src, regclass:$r0, regclass:$r1,
7554                                 regclass:$r2, regclass:$r3);
7555   dag InsR47 = (ins regclass:$r4, regclass:$r5,
7556                     regclass:$r6, regclass:$r7);
7557   dag InsR = !if(!eq(Type,"f16"), InsR03, !con(InsR03, InsR47));
7558   dag StrideArg = !if(WithStride, (ins Int32Regs:$ldm), (ins));
7559   dag Ins = !con(InsR, StrideArg);
7560
7561   // Construct the pattern to match corresponding intrinsic call. See the
7562   // details in the comments in WMMA_LOAD_ALSTOS.
7563   dag PatArgs = !foreach(tmp, Ins,
7564                               !subst(imem, ADDRvar,
7565                               !subst(MEMri64, ADDRri64,
7566                               !subst(MEMri, ADDRri,
7567                               !subst(ins, IntrMatcher, tmp)))));
7568   let Pattern = [PatArgs];
7569   let OutOperandList = (outs);
7570   let InOperandList = Ins;
7571   let AsmString = "wmma.store.d.sync."
7572                   # Layout
7573                   # "." # Geometry
7574                   # Space
7575                   # "." # Type
7576                   # " \t[$src],"
7577                   # !if(!eq(Type,"f16"),
7578                         "{{$r0, $r1, $r2, $r3}}",
7579                         "{{$r0, $r1, $r2, $r3, $r4, $r5, $r6, $r7}}")
7580                   # !if(WithStride, ", $ldm", "")
7581                   # ";";
7582
7583 }
7584
7585 class WMMA_STORE_INTR_HELPER<string Geometry, string Layout, string Space,
7586                              string Type, bit WithStride>
7587                             : PatFrag <(ops),(ops)> {
7588   // Intrinsic that matches this instruction.
7589   Intrinsic Intr = !cast<Intrinsic>("int_nvvm_wmma_"
7590                                     # Geometry
7591                                     # "_store_d"
7592                                     # "_" # Type
7593                                     # "_" # Layout
7594                                     # !if(WithStride, "_stride", ""));
7595   code match_generic = [{
7596    return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_GENERIC);
7597   }];
7598   code match_shared = [{
7599    return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_SHARED);
7600   }];
7601   code match_global = [{
7602    return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_GLOBAL);
7603   }];
7604
7605   dag Args = !if(!eq(Type,"f16"),
7606                  (ops node:$dst, node:$r0, node:$r1, node:$r2, node:$r3),
7607                  (ops node:$dst, node:$r0, node:$r1, node:$r2, node:$r3,
7608                                  node:$r4, node:$r5, node:$r6, node:$r7));
7609   dag StrideArg = !if(WithStride, (ops node:$ldm), (ops));
7610   let Operands = !con(Args, StrideArg);
7611   let Fragments = [!foreach(tmp, Operands, !subst(ops, Intr, tmp))];
7612   let PredicateCode = !if(!eq(Space, ".shared"), match_shared,
7613                       !if(!eq(Space, ".global"), match_global, match_generic));
7614 }
7615
7616 multiclass WMMA_STORE_D_GLSTS<string Geometry, string Layout, string Space,
7617                               string Type, NVPTXRegClass regclass,
7618                               bit WithStride> {
7619   def _avar:   WMMA_STORE_D_GLSTSO<Geometry, Layout, Space, Type, regclass,
7620                                    WithStride, imem>;
7621   def _areg:   WMMA_STORE_D_GLSTSO<Geometry, Layout, Space, Type, regclass,
7622                                    WithStride, Int32Regs>;
7623   def _areg64: WMMA_STORE_D_GLSTSO<Geometry, Layout, Space, Type, regclass,
7624                                    WithStride, Int64Regs>;
7625   def _ari:    WMMA_STORE_D_GLSTSO<Geometry, Layout, Space, Type, regclass,
7626                                    WithStride, MEMri>;
7627   def _ari64:  WMMA_STORE_D_GLSTSO<Geometry, Layout, Space, Type, regclass,
7628                                    WithStride, MEMri64>;
7629 }
7630
7631 multiclass WMMA_STORE_D_GLSTSh<string Geometry, string Layout, string Space,
7632                                string Type, NVPTXRegClass regclass,
7633                                bit WithStride> {
7634   // Define a PatFrag that matches appropriate intrinsic that loads from the
7635   // given address space.
7636   def _Intr:    WMMA_STORE_INTR_HELPER<Geometry, Layout, Space, Type,
7637                                        WithStride>;
7638   defm NAME:    WMMA_STORE_D_GLSTS<Geometry, Layout, Space, Type, regclass,
7639                                    WithStride>;
7640 }
7641
7642 multiclass WMMA_STORE_D_GLST<string Geometry, string Layout, string Space,
7643                              string Type, NVPTXRegClass regclass > {
7644   defm _stride: WMMA_STORE_D_GLSTSh<Geometry, Layout, Space, Type, regclass, 1>;
7645   defm NAME:    WMMA_STORE_D_GLSTSh<Geometry, Layout, Space, Type, regclass, 0>;
7646 }
7647
7648 multiclass WMMA_STORE_D_GLT<string Geometry, string Layout,
7649                            string Type, NVPTXRegClass regclass> {
7650   defm _global: WMMA_STORE_D_GLST<Geometry, Layout, ".global", Type, regclass>;
7651   defm _shared: WMMA_STORE_D_GLST<Geometry, Layout, ".shared", Type, regclass>;
7652   defm NAME:    WMMA_STORE_D_GLST<Geometry, Layout,        "", Type, regclass>;
7653 }
7654
7655 multiclass WMMA_STORE_D_GT<string Geometry, string Type,
7656                            NVPTXRegClass regclass> {
7657   defm _row:    WMMA_STORE_D_GLT<Geometry, "row", Type, regclass>;
7658   defm _col:    WMMA_STORE_D_GLT<Geometry, "col", Type, regclass>;
7659 }
7660
7661 multiclass WMMA_STORE_D_G<string Geometry> {
7662   defm _store_d_f16: WMMA_STORE_D_GT<Geometry, "f16", Float16x2Regs>;
7663   defm _store_d_f32: WMMA_STORE_D_GT<Geometry, "f32", Float32Regs>;
7664 }
7665
7666 defm INT_WMMA_m32n8k16: WMMA_STORE_D_G<"m32n8k16">;
7667 defm INT_WMMA_m16n16k16: WMMA_STORE_D_G<"m16n16k16">;
7668 defm INT_WMMA_m8n32k16: WMMA_STORE_D_G<"m8n32k16">;
7669
7670 // WMMA.MMA
7671 class WMMA_MMA_GABDCS<string Geometry, string ALayout, string BLayout,
7672                      string DType, NVPTXRegClass d_reg,
7673                      string CType, NVPTXRegClass c_reg,
7674                      NVPTXRegClass ab_reg,
7675                      string Satfinite = "">
7676   : EmptyNVPTXInst,
7677     Requires<[!if(!eq(Geometry, "m16n16k16"),
7678                   hasPTX60,
7679                   hasPTX61),
7680               hasSM70]> {
7681   Intrinsic Intr = !cast<Intrinsic>("int_nvvm_wmma_"
7682                                     # Geometry
7683                                     # "_mma"
7684                                     # "_" # ALayout
7685                                     # "_" # BLayout
7686                                     # "_" # DType
7687                                     # "_" # CType
7688                                     # !subst(".", "_", Satfinite));
7689   dag Outs = !if(!eq(DType,"f16"),
7690                  (outs d_reg:$d0, d_reg:$d1, d_reg:$d2, d_reg:$d3),
7691                  (outs d_reg:$d0, d_reg:$d1, d_reg:$d2, d_reg:$d3,
7692                        d_reg:$d4, d_reg:$d5, d_reg:$d6, d_reg:$d7));
7693   dag InsExtraCArgs = !if(!eq(CType,"f16"),
7694                           (ins),
7695                           (ins c_reg:$c4,  c_reg:$c5,  c_reg:$c6,  c_reg:$c7));
7696   dag Ins = !con((ins ab_reg:$a0, ab_reg:$a1, ab_reg:$a2, ab_reg:$a3,
7697                       ab_reg:$a4, ab_reg:$a5, ab_reg:$a6, ab_reg:$a7,
7698                       ab_reg:$b0, ab_reg:$b1, ab_reg:$b2, ab_reg:$b3,
7699                       ab_reg:$b4, ab_reg:$b5, ab_reg:$b6, ab_reg:$b7,
7700                       c_reg:$c0,  c_reg:$c1,  c_reg:$c2,  c_reg:$c3),
7701                   InsExtraCArgs);
7702
7703   // Construct the pattern to match corresponding intrinsic call. See the
7704   // details in the comments in WMMA_LOAD_ALSTOS.
7705   dag PatOuts = !foreach(tmp, Outs, !subst(outs, set, tmp));
7706   dag PatArgs = !foreach(tmp, Ins, !subst(ins, Intr, tmp));
7707   let Pattern = [!con(PatOuts, (set PatArgs))];
7708   let OutOperandList = Outs;
7709   let InOperandList  = Ins;
7710   let AsmString = "wmma.mma.sync."
7711                   # ALayout
7712                   # "." # BLayout
7713                   # "." # Geometry
7714                   # "." # DType
7715                   # "." # CType
7716                   # Satfinite # "\n\t\t"
7717                   # !if(!eq(DType,"f16"),
7718                         "{{$d0, $d1, $d2, $d3}}, \n\t\t",
7719                         "{{$d0, $d1, $d2, $d3, $d4, $d5, $d6, $d7}},\n\t\t")
7720                   # "{{$a0, $a1, $a2, $a3, $a4, $a5, $a6, $a7}},\n\t\t"
7721                   # "{{$b0, $b1, $b2, $b3, $b4, $b5, $b6, $b7}},\n\t\t"
7722                   # !if(!eq(CType,"f16"),
7723                         "{{$c0, $c1, $c2, $c3}};",
7724                         "{{$c0, $c1, $c2, $c3, $c4, $c5, $c6, $c7}};");
7725 }
7726
7727 multiclass WMMA_MMA_GABDC<string Geometry, string ALayout, string BLayout,
7728                          string DType, NVPTXRegClass d_reg,
7729                          string CType, NVPTXRegClass c_reg> {
7730   def _satfinite: WMMA_MMA_GABDCS<Geometry, ALayout, BLayout,
7731                                  DType, d_reg, CType, c_reg,
7732                                  Float16x2Regs, ".satfinite">;
7733   def NAME:       WMMA_MMA_GABDCS<Geometry, ALayout, BLayout,
7734                                  DType, d_reg, CType, c_reg,
7735                                  Float16x2Regs>;
7736 }
7737
7738 multiclass WMMA_MMA_GABD<string Geometry, string ALayout, string BLayout,
7739                         string DType, NVPTXRegClass d_reg> {
7740   defm _f16: WMMA_MMA_GABDC<Geometry, ALayout, BLayout, DType, d_reg,
7741                             "f16", Float16x2Regs>;
7742   defm _f32: WMMA_MMA_GABDC<Geometry, ALayout, BLayout, DType, d_reg,
7743                             "f32", Float32Regs>;
7744 }
7745
7746 multiclass WMMA_MMA_GAB<string Geometry, string ALayout, string BLayout> {
7747   defm _f16: WMMA_MMA_GABD<Geometry, ALayout, BLayout, "f16", Float16x2Regs>;
7748   defm _f32: WMMA_MMA_GABD<Geometry, ALayout, BLayout, "f32", Float32Regs>;
7749 }
7750
7751 multiclass WMMA_MMA_GA<string Geometry, string ALayout> {
7752   defm _col: WMMA_MMA_GAB<Geometry, ALayout, "col">;
7753   defm _row: WMMA_MMA_GAB<Geometry, ALayout, "row">;
7754 }
7755
7756 multiclass WMMA_MMA_G<string Geometry> {
7757   defm _col: WMMA_MMA_GA<Geometry, "col">;
7758   defm _row: WMMA_MMA_GA<Geometry, "row">;
7759 }
7760
7761 defm INT_WMMA_MMA_m32n8k16 : WMMA_MMA_G<"m32n8k16">;
7762 defm INT_WMMA_MMA_m16n16k16 : WMMA_MMA_G<"m16n16k16">;
7763 defm INT_WMMA_MMA_m8n32k16 : WMMA_MMA_G<"m8n32k16">;