contrib/llvm/lib/Target/AMDGPU/BUFInstructions.td

   1 //===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9
  10 def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">;
  11 def MUBUFAddr64 : ComplexPattern<i64, 7, "SelectMUBUFAddr64">;
  12 def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">;
  13
  14 def MUBUFScratch : ComplexPattern<i64, 4, "SelectMUBUFScratch">;
  15 def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">;
  16 def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
  17 def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">;
  18 def MUBUFIntrinsicOffset : ComplexPattern<i32, 2, "SelectMUBUFIntrinsicOffset">;
  19 def MUBUFIntrinsicVOffset : ComplexPattern<i32, 3, "SelectMUBUFIntrinsicVOffset">;
  20
  21 class MubufLoad <SDPatternOperator op> : PatFrag <
  22   (ops node:$ptr), (op node:$ptr), [{
  23   auto const AS = cast<MemSDNode>(N)->getAddressSpace();
  24   return AS == AMDGPUAS::GLOBAL_ADDRESS ||
  25          AS == AMDGPUAS::CONSTANT_ADDRESS;
  26 }]>;
  27
  28 def mubuf_load          : MubufLoad <load>;
  29 def mubuf_az_extloadi8  : MubufLoad <az_extloadi8>;
  30 def mubuf_sextloadi8    : MubufLoad <sextloadi8>;
  31 def mubuf_az_extloadi16 : MubufLoad <az_extloadi16>;
  32 def mubuf_sextloadi16   : MubufLoad <sextloadi16>;
  33 def mubuf_load_atomic   : MubufLoad <atomic_load>;
  34
  35 def BUFAddrKind {
  36   int Offset = 0;
  37   int OffEn  = 1;
  38   int IdxEn  = 2;
  39   int BothEn = 3;
  40   int Addr64 = 4;
  41 }
  42
  43 class getAddrName<int addrKind> {
  44   string ret =
  45     !if(!eq(addrKind, BUFAddrKind.Offset), "offset",
  46     !if(!eq(addrKind, BUFAddrKind.OffEn),  "offen",
  47     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "idxen",
  48     !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",
  49     !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",
  50     "")))));
  51 }
  52
  53 class MUBUFAddr64Table <bit is_addr64, string suffix = ""> {
  54   bit IsAddr64 = is_addr64;
  55   string OpName = NAME # suffix;
  56 }
  57
  58 //===----------------------------------------------------------------------===//
  59 // MTBUF classes
  60 //===----------------------------------------------------------------------===//
  61
  62 class MTBUF_Pseudo <string opName, dag outs, dag ins,
  63                     string asmOps, list<dag> pattern=[]> :
  64   InstSI<outs, ins, "", pattern>,
  65   SIMCInstr<opName, SIEncodingFamily.NONE> {
  66
  67   let isPseudo = 1;
  68   let isCodeGenOnly = 1;
  69   let Size = 8;
  70   let UseNamedOperandTable = 1;
  71
  72   string Mnemonic = opName;
  73   string AsmOperands = asmOps;
  74
  75   let VM_CNT = 1;
  76   let EXP_CNT = 1;
  77   let MTBUF = 1;
  78   let Uses = [EXEC];
  79
  80   let hasSideEffects = 0;
  81   let SchedRW = [WriteVMEM];
  82 }
  83
  84 class MTBUF_Real <MTBUF_Pseudo ps> :
  85   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>,
  86   Enc64 {
  87
  88   let isPseudo = 0;
  89   let isCodeGenOnly = 0;
  90
  91   // copy relevant pseudo op flags
  92   let SubtargetPredicate = ps.SubtargetPredicate;
  93   let AsmMatchConverter  = ps.AsmMatchConverter;
  94   let Constraints        = ps.Constraints;
  95   let DisableEncoding    = ps.DisableEncoding;
  96   let TSFlags            = ps.TSFlags;
  97
  98   bits<8> vdata;
  99   bits<12> offset;
 100   bits<1> offen;
 101   bits<1> idxen;
 102   bits<1> glc;
 103   bits<1> addr64;
 104   bits<4> dfmt;
 105   bits<3> nfmt;
 106   bits<8> vaddr;
 107   bits<7> srsrc;
 108   bits<1> slc;
 109   bits<1> tfe;
 110   bits<8> soffset;
 111
 112   let Inst{11-0}  = offset;
 113   let Inst{12}    = offen;
 114   let Inst{13}    = idxen;
 115   let Inst{14}    = glc;
 116   let Inst{22-19} = dfmt;
 117   let Inst{25-23} = nfmt;
 118   let Inst{31-26} = 0x3a; //encoding
 119   let Inst{39-32} = vaddr;
 120   let Inst{47-40} = vdata;
 121   let Inst{52-48} = srsrc{6-2};
 122   let Inst{54}    = slc;
 123   let Inst{55}    = tfe;
 124   let Inst{63-56} = soffset;
 125 }
 126
 127 class MTBUF_Load_Pseudo <string opName, RegisterClass regClass> : MTBUF_Pseudo <
 128   opName, (outs regClass:$dst),
 129   (ins u16imm:$offset, i1imm:$offen, i1imm:$idxen, i1imm:$glc, i1imm:$addr64,
 130        i8imm:$dfmt, i8imm:$nfmt, VGPR_32:$vaddr, SReg_128:$srsrc,
 131        i1imm:$slc, i1imm:$tfe, SCSrc_b32:$soffset),
 132   " $dst, $offset, $offen, $idxen, $glc, $addr64, $dfmt,"#
 133   " $nfmt, $vaddr, $srsrc, $slc, $tfe, $soffset"> {
 134   let mayLoad = 1;
 135   let mayStore = 0;
 136 }
 137
 138 class MTBUF_Store_Pseudo <string opName, RegisterClass regClass> : MTBUF_Pseudo <
 139   opName, (outs),
 140   (ins regClass:$vdata, u16imm:$offset, i1imm:$offen, i1imm:$idxen, i1imm:$glc,
 141        i1imm:$addr64, i8imm:$dfmt, i8imm:$nfmt, VGPR_32:$vaddr,
 142        SReg_128:$srsrc, i1imm:$slc, i1imm:$tfe, SCSrc_b32:$soffset),
 143   " $vdata, $offset, $offen, $idxen, $glc, $addr64, $dfmt,"#
 144   " $nfmt, $vaddr, $srsrc, $slc, $tfe, $soffset"> {
 145   let mayLoad = 0;
 146   let mayStore = 1;
 147 }
 148
 149 //===----------------------------------------------------------------------===//
 150 // MUBUF classes
 151 //===----------------------------------------------------------------------===//
 152
 153 class MUBUF_Pseudo <string opName, dag outs, dag ins,
 154                     string asmOps, list<dag> pattern=[]> :
 155   InstSI<outs, ins, "", pattern>,
 156   SIMCInstr<opName, SIEncodingFamily.NONE> {
 157
 158   let isPseudo = 1;
 159   let isCodeGenOnly = 1;
 160   let Size = 8;
 161   let UseNamedOperandTable = 1;
 162
 163   string Mnemonic = opName;
 164   string AsmOperands = asmOps;
 165
 166   let VM_CNT = 1;
 167   let EXP_CNT = 1;
 168   let MUBUF = 1;
 169   let Uses = [EXEC];
 170   let hasSideEffects = 0;
 171   let SchedRW = [WriteVMEM];
 172
 173   let AsmMatchConverter = "cvtMubuf";
 174
 175   bits<1> offen       = 0;
 176   bits<1> idxen       = 0;
 177   bits<1> addr64      = 0;
 178   bits<1> has_vdata   = 1;
 179   bits<1> has_vaddr   = 1;
 180   bits<1> has_glc     = 1;
 181   bits<1> glc_value   = 0; // the value for glc if no such operand
 182   bits<1> has_srsrc   = 1;
 183   bits<1> has_soffset = 1;
 184   bits<1> has_offset  = 1;
 185   bits<1> has_slc     = 1;
 186   bits<1> has_tfe     = 1;
 187 }
 188
 189 class MUBUF_Real <bits<7> op, MUBUF_Pseudo ps> :
 190   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 191
 192   let isPseudo = 0;
 193   let isCodeGenOnly = 0;
 194
 195   // copy relevant pseudo op flags
 196   let SubtargetPredicate = ps.SubtargetPredicate;
 197   let AsmMatchConverter  = ps.AsmMatchConverter;
 198   let Constraints        = ps.Constraints;
 199   let DisableEncoding    = ps.DisableEncoding;
 200   let TSFlags            = ps.TSFlags;
 201
 202   bits<12> offset;
 203   bits<1>  glc;
 204   bits<1>  lds = 0;
 205   bits<8>  vaddr;
 206   bits<8>  vdata;
 207   bits<7>  srsrc;
 208   bits<1>  slc;
 209   bits<1>  tfe;
 210   bits<8>  soffset;
 211 }
 212
 213
 214 // For cache invalidation instructions.
 215 class MUBUF_Invalidate <string opName, SDPatternOperator node> :
 216   MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {
 217
 218   let AsmMatchConverter = "";
 219
 220   let hasSideEffects = 1;
 221   let mayStore = 1;
 222
 223   // Set everything to 0.
 224   let offen       = 0;
 225   let idxen       = 0;
 226   let addr64      = 0;
 227   let has_vdata   = 0;
 228   let has_vaddr   = 0;
 229   let has_glc     = 0;
 230   let glc_value   = 0;
 231   let has_srsrc   = 0;
 232   let has_soffset = 0;
 233   let has_offset  = 0;
 234   let has_slc     = 0;
 235   let has_tfe     = 0;
 236 }
 237
 238 class getMUBUFInsDA<list<RegisterClass> vdataList,
 239                     list<RegisterClass> vaddrList=[]> {
 240   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 241   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 242   dag InsNoData = !if(!empty(vaddrList),
 243     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 244          offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe),
 245     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 246          offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe)
 247   );
 248   dag InsData = !if(!empty(vaddrList),
 249     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 250          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe),
 251     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 252          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe)
 253   );
 254   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 255 }
 256
 257 class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
 258   dag ret =
 259     !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList>.ret,
 260     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMUBUFInsDA<vdataList, [VGPR_32]>.ret,
 261     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMUBUFInsDA<vdataList, [VGPR_32]>.ret,
 262     !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64]>.ret,
 263     !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64]>.ret,
 264     (ins))))));
 265 }
 266
 267 class getMUBUFAsmOps<int addrKind> {
 268   string Pfx =
 269     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",
 270     !if(!eq(addrKind, BUFAddrKind.OffEn),  "$vaddr, $srsrc, $soffset offen",
 271     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "$vaddr, $srsrc, $soffset idxen",
 272     !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",
 273     !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",
 274     "")))));
 275   string ret = Pfx # "$offset";
 276 }
 277
 278 class MUBUF_SetupAddr<int addrKind> {
 279   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 280                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 281
 282   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 283                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 284
 285   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 286
 287   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 288 }
 289
 290 class MUBUF_Load_Pseudo <string opName,
 291                          int addrKind,
 292                          RegisterClass vdataClass,
 293                          list<dag> pattern=[],
 294                          // Workaround bug bz30254
 295                          int addrKindCopy = addrKind>
 296   : MUBUF_Pseudo<opName,
 297                  (outs vdataClass:$vdata),
 298                  getMUBUFIns<addrKindCopy>.ret,
 299                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
 300                  pattern>,
 301     MUBUF_SetupAddr<addrKindCopy> {
 302   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 303   let mayLoad = 1;
 304   let mayStore = 0;
 305 }
 306
 307 // FIXME: tfe can't be an operand because it requires a separate
 308 // opcode because it needs an N+1 register class dest register.
 309 multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 310                               ValueType load_vt = i32,
 311                               SDPatternOperator ld = null_frag> {
 312
 313   def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 314     [(set load_vt:$vdata,
 315      (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>,
 316     MUBUFAddr64Table<0>;
 317
 318   def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 319     [(set load_vt:$vdata,
 320      (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>,
 321     MUBUFAddr64Table<1>;
 322
 323   def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 324   def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 325   def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 326
 327   let DisableWQM = 1 in {
 328     def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 329     def _OFFEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 330     def _IDXEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 331     def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 332   }
 333 }
 334
 335 class MUBUF_Store_Pseudo <string opName,
 336                           int addrKind,
 337                           RegisterClass vdataClass,
 338                           list<dag> pattern=[],
 339                           // Workaround bug bz30254
 340                           int addrKindCopy = addrKind,
 341                           RegisterClass vdataClassCopy = vdataClass>
 342   : MUBUF_Pseudo<opName,
 343                  (outs),
 344                  getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 345                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
 346                  pattern>,
 347     MUBUF_SetupAddr<addrKindCopy> {
 348   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 349   let mayLoad = 0;
 350   let mayStore = 1;
 351 }
 352
 353 multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 354                                ValueType store_vt = i32,
 355                                SDPatternOperator st = null_frag> {
 356
 357   def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 358     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 359                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>,
 360     MUBUFAddr64Table<0>;
 361
 362   def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 363     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 364                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>,
 365     MUBUFAddr64Table<1>;
 366
 367   def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 368   def _IDXEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 369   def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 370
 371   let DisableWQM = 1 in {
 372     def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 373     def _OFFEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 374     def _IDXEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 375     def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 376   }
 377 }
 378
 379
 380 class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
 381                           list<RegisterClass> vaddrList=[]> {
 382   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 383   dag ret = !if(vdata_in,
 384     !if(!empty(vaddrList),
 385       (ins vdataClass:$vdata_in,
 386            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc),
 387       (ins vdataClass:$vdata_in, vaddrClass:$vaddr,
 388            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc)
 389     ),
 390     !if(!empty(vaddrList),
 391       (ins vdataClass:$vdata,
 392            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc),
 393       (ins vdataClass:$vdata, vaddrClass:$vaddr,
 394            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc)
 395   ));
 396 }
 397
 398 class getMUBUFAtomicIns<int addrKind,
 399                         RegisterClass vdataClass,
 400                         bit vdata_in,
 401                         // Workaround bug bz30254
 402                         RegisterClass vdataClassCopy=vdataClass> {
 403   dag ret =
 404     !if(!eq(addrKind, BUFAddrKind.Offset),
 405             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret,
 406     !if(!eq(addrKind, BUFAddrKind.OffEn),
 407             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 408     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 409             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 410     !if(!eq(addrKind, BUFAddrKind.BothEn),
 411             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 412     !if(!eq(addrKind, BUFAddrKind.Addr64),
 413             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 414     (ins))))));
 415 }
 416
 417 class MUBUF_Atomic_Pseudo<string opName,
 418                           int addrKind,
 419                           dag outs,
 420                           dag ins,
 421                           string asmOps,
 422                           list<dag> pattern=[],
 423                           // Workaround bug bz30254
 424                           int addrKindCopy = addrKind>
 425   : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,
 426     MUBUF_SetupAddr<addrKindCopy> {
 427   let mayStore = 1;
 428   let mayLoad = 1;
 429   let hasPostISelHook = 1;
 430   let hasSideEffects = 1;
 431   let DisableWQM = 1;
 432   let has_glc = 0;
 433   let has_tfe = 0;
 434 }
 435
 436 class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
 437                                RegisterClass vdataClass,
 438                                list<dag> pattern=[],
 439                                // Workaround bug bz30254
 440                                int addrKindCopy = addrKind,
 441                                RegisterClass vdataClassCopy = vdataClass>
 442   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 443                         (outs),
 444                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret,
 445                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc",
 446                         pattern>,
 447     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> {
 448   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 449   let glc_value = 0;
 450   let AsmMatchConverter = "cvtMubufAtomic";
 451 }
 452
 453 class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
 454                              RegisterClass vdataClass,
 455                              list<dag> pattern=[],
 456                              // Workaround bug bz30254
 457                              int addrKindCopy = addrKind,
 458                              RegisterClass vdataClassCopy = vdataClass>
 459   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 460                         (outs vdataClassCopy:$vdata),
 461                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret,
 462                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc",
 463                         pattern>,
 464     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> {
 465   let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret;
 466   let glc_value = 1;
 467   let Constraints = "$vdata = $vdata_in";
 468   let DisableEncoding = "$vdata_in";
 469   let AsmMatchConverter = "cvtMubufAtomicReturn";
 470 }
 471
 472 multiclass MUBUF_Pseudo_Atomics <string opName,
 473                                  RegisterClass vdataClass,
 474                                  ValueType vdataType,
 475                                  SDPatternOperator atomic> {
 476
 477   def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>,
 478                 MUBUFAddr64Table <0>;
 479   def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>,
 480                 MUBUFAddr64Table <1>;
 481   def _OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 482   def _IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 483   def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 484
 485   def _RTN_OFFSET : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 486     [(set vdataType:$vdata,
 487      (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc),
 488              vdataType:$vdata_in))]>,
 489     MUBUFAddr64Table <0, "_RTN">;
 490
 491   def _RTN_ADDR64 : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 492     [(set vdataType:$vdata,
 493      (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc),
 494              vdataType:$vdata_in))]>,
 495     MUBUFAddr64Table <1, "_RTN">;
 496
 497   def _RTN_OFFEN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 498   def _RTN_IDXEN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 499   def _RTN_BOTHEN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 500 }
 501
 502
 503 //===----------------------------------------------------------------------===//
 504 // MUBUF Instructions
 505 //===----------------------------------------------------------------------===//
 506
 507 let SubtargetPredicate = isGCN in {
 508
 509 defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads <
 510   "buffer_load_format_x", VGPR_32
 511 >;
 512 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
 513   "buffer_load_format_xy", VReg_64
 514 >;
 515 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <
 516   "buffer_load_format_xyz", VReg_96
 517 >;
 518 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <
 519   "buffer_load_format_xyzw", VReg_128
 520 >;
 521 defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <
 522   "buffer_store_format_x", VGPR_32
 523 >;
 524 defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <
 525   "buffer_store_format_xy", VReg_64
 526 >;
 527 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <
 528   "buffer_store_format_xyz", VReg_96
 529 >;
 530 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <
 531   "buffer_store_format_xyzw", VReg_128
 532 >;
 533 defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads <
 534   "buffer_load_ubyte", VGPR_32, i32, mubuf_az_extloadi8
 535 >;
 536 defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads <
 537   "buffer_load_sbyte", VGPR_32, i32, mubuf_sextloadi8
 538 >;
 539 defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads <
 540   "buffer_load_ushort", VGPR_32, i32, mubuf_az_extloadi16
 541 >;
 542 defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads <
 543   "buffer_load_sshort", VGPR_32, i32, mubuf_sextloadi16
 544 >;
 545 defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads <
 546   "buffer_load_dword", VGPR_32, i32, mubuf_load
 547 >;
 548 defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
 549   "buffer_load_dwordx2", VReg_64, v2i32, mubuf_load
 550 >;
 551 defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
 552   "buffer_load_dwordx3", VReg_96, untyped, mubuf_load
 553 >;
 554 defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
 555   "buffer_load_dwordx4", VReg_128, v4i32, mubuf_load
 556 >;
 557 defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
 558   "buffer_store_byte", VGPR_32, i32, truncstorei8_global
 559 >;
 560 defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
 561   "buffer_store_short", VGPR_32, i32, truncstorei16_global
 562 >;
 563 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
 564   "buffer_store_dword", VGPR_32, i32, global_store
 565 >;
 566 defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
 567   "buffer_store_dwordx2", VReg_64, v2i32, global_store
 568 >;
 569 defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
 570   "buffer_store_dwordx3", VReg_96, untyped, global_store
 571 >;
 572 defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
 573   "buffer_store_dwordx4", VReg_128, v4i32, global_store
 574 >;
 575 defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
 576   "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global
 577 >;
 578 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <
 579   "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag
 580 >;
 581 defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <
 582   "buffer_atomic_add", VGPR_32, i32, atomic_add_global
 583 >;
 584 defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <
 585   "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global
 586 >;
 587 defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <
 588   "buffer_atomic_smin", VGPR_32, i32, atomic_min_global
 589 >;
 590 defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <
 591   "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global
 592 >;
 593 defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <
 594   "buffer_atomic_smax", VGPR_32, i32, atomic_max_global
 595 >;
 596 defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <
 597   "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global
 598 >;
 599 defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <
 600   "buffer_atomic_and", VGPR_32, i32, atomic_and_global
 601 >;
 602 defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <
 603   "buffer_atomic_or", VGPR_32, i32, atomic_or_global
 604 >;
 605 defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <
 606   "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global
 607 >;
 608 defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <
 609   "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global
 610 >;
 611 defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <
 612   "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global
 613 >;
 614 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <
 615   "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global
 616 >;
 617 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <
 618   "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag
 619 >;
 620 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <
 621   "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global
 622 >;
 623 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <
 624   "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global
 625 >;
 626 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <
 627   "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global
 628 >;
 629 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <
 630   "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global
 631 >;
 632 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <
 633   "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global
 634 >;
 635 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <
 636   "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global
 637 >;
 638 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <
 639   "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global
 640 >;
 641 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <
 642   "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global
 643 >;
 644 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <
 645   "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global
 646 >;
 647 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <
 648   "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global
 649 >;
 650 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
 651   "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global
 652 >;
 653
 654 let SubtargetPredicate = isSI in { // isn't on CI & VI
 655 /*
 656 defm BUFFER_ATOMIC_RSUB        : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
 657 defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">;
 658 defm BUFFER_ATOMIC_FMIN        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">;
 659 defm BUFFER_ATOMIC_FMAX        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">;
 660 defm BUFFER_ATOMIC_RSUB_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
 661 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">;
 662 defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">;
 663 defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">;
 664 */
 665
 666 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
 667                                           int_amdgcn_buffer_wbinvl1_sc>;
 668 }
 669
 670 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
 671                                        int_amdgcn_buffer_wbinvl1>;
 672
 673 //===----------------------------------------------------------------------===//
 674 // MTBUF Instructions
 675 //===----------------------------------------------------------------------===//
 676
 677 //def TBUFFER_LOAD_FORMAT_X    : MTBUF_ <0, "tbuffer_load_format_x", []>;
 678 //def TBUFFER_LOAD_FORMAT_XY   : MTBUF_ <1, "tbuffer_load_format_xy", []>;
 679 //def TBUFFER_LOAD_FORMAT_XYZ  : MTBUF_ <2, "tbuffer_load_format_xyz", []>;
 680 def TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Load_Pseudo  <"tbuffer_load_format_xyzw", VReg_128>;
 681 def TBUFFER_STORE_FORMAT_X    : MTBUF_Store_Pseudo <"tbuffer_store_format_x", VGPR_32>;
 682 def TBUFFER_STORE_FORMAT_XY   : MTBUF_Store_Pseudo <"tbuffer_store_format_xy", VReg_64>;
 683 def TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Store_Pseudo <"tbuffer_store_format_xyz", VReg_128>;
 684 def TBUFFER_STORE_FORMAT_XYZW : MTBUF_Store_Pseudo <"tbuffer_store_format_xyzw", VReg_128>;
 685
 686 } // End let SubtargetPredicate = isGCN
 687
 688 let SubtargetPredicate = isCIVI in {
 689
 690 //===----------------------------------------------------------------------===//
 691 // Instruction definitions for CI and newer.
 692 //===----------------------------------------------------------------------===//
 693 // Remaining instructions:
 694 // BUFFER_LOAD_DWORDX3
 695 // BUFFER_STORE_DWORDX3
 696
 697 def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
 698                                            int_amdgcn_buffer_wbinvl1_vol>;
 699
 700 } // End let SubtargetPredicate = isCIVI
 701
 702 //===----------------------------------------------------------------------===//
 703 // MUBUF Patterns
 704 //===----------------------------------------------------------------------===//
 705
 706 let Predicates = [isGCN] in {
 707
 708 // int_SI_vs_load_input
 709 def : Pat<
 710   (SIload_input v4i32:$tlst, imm:$attr_offset, i32:$buf_idx_vgpr),
 711   (BUFFER_LOAD_FORMAT_XYZW_IDXEN $buf_idx_vgpr, $tlst, (i32 0), imm:$attr_offset, 0, 0, 0)
 712 >;
 713
 714 // Offset in an 32-bit VGPR
 715 def : Pat <
 716   (SIload_constant v4i32:$sbase, i32:$voff),
 717   (BUFFER_LOAD_DWORD_OFFEN $voff, $sbase, (i32 0), 0, 0, 0, 0)
 718 >;
 719
 720
 721 //===----------------------------------------------------------------------===//
 722 // buffer_load/store_format patterns
 723 //===----------------------------------------------------------------------===//
 724
 725 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
 726                                   string opcode> {
 727   def : Pat<
 728     (vt (name v4i32:$rsrc, 0,
 729               (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 730               imm:$glc, imm:$slc)),
 731     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
 732       (as_i1imm $glc), (as_i1imm $slc), 0)
 733   >;
 734
 735   def : Pat<
 736     (vt (name v4i32:$rsrc, i32:$vindex,
 737               (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 738               imm:$glc, imm:$slc)),
 739     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
 740       (as_i1imm $glc), (as_i1imm $slc), 0)
 741   >;
 742
 743   def : Pat<
 744     (vt (name v4i32:$rsrc, 0,
 745               (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 746               imm:$glc, imm:$slc)),
 747     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
 748       (as_i1imm $glc), (as_i1imm $slc), 0)
 749   >;
 750
 751   def : Pat<
 752     (vt (name v4i32:$rsrc, i32:$vindex,
 753               (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 754               imm:$glc, imm:$slc)),
 755     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
 756       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
 757       $rsrc, $soffset, (as_i16imm $offset),
 758       (as_i1imm $glc), (as_i1imm $slc), 0)
 759   >;
 760 }
 761
 762 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
 763 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
 764 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;
 765 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">;
 766 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">;
 767 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">;
 768
 769 multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
 770                                    string opcode> {
 771   def : Pat<
 772     (name vt:$vdata, v4i32:$rsrc, 0,
 773           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 774           imm:$glc, imm:$slc),
 775     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset),
 776                                     (as_i1imm $glc), (as_i1imm $slc), 0)
 777   >;
 778
 779   def : Pat<
 780     (name vt:$vdata, v4i32:$rsrc, i32:$vindex,
 781           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 782           imm:$glc, imm:$slc),
 783     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
 784                                    (as_i16imm $offset), (as_i1imm $glc),
 785                                    (as_i1imm $slc), 0)
 786   >;
 787
 788   def : Pat<
 789     (name vt:$vdata, v4i32:$rsrc, 0,
 790           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 791           imm:$glc, imm:$slc),
 792     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
 793                                    (as_i16imm $offset), (as_i1imm $glc),
 794                                    (as_i1imm $slc), 0)
 795   >;
 796
 797   def : Pat<
 798     (name vt:$vdata, v4i32:$rsrc, i32:$vindex,
 799           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 800           imm:$glc, imm:$slc),
 801     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
 802       $vdata,
 803       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
 804       $rsrc, $soffset, (as_i16imm $offset),
 805       (as_i1imm $glc), (as_i1imm $slc), 0)
 806   >;
 807 }
 808
 809 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
 810 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
 811 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
 812 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, f32, "BUFFER_STORE_DWORD">;
 813 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v2f32, "BUFFER_STORE_DWORDX2">;
 814 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v4f32, "BUFFER_STORE_DWORDX4">;
 815
 816 //===----------------------------------------------------------------------===//
 817 // buffer_atomic patterns
 818 //===----------------------------------------------------------------------===//
 819
 820 multiclass BufferAtomicPatterns<SDPatternOperator name, string opcode> {
 821   def : Pat<
 822     (name i32:$vdata_in, v4i32:$rsrc, 0,
 823           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 824           imm:$slc),
 825     (!cast<MUBUF_Pseudo>(opcode # _RTN_OFFSET) $vdata_in, $rsrc, $soffset,
 826                                         (as_i16imm $offset), (as_i1imm $slc))
 827   >;
 828
 829   def : Pat<
 830     (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex,
 831           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 832           imm:$slc),
 833     (!cast<MUBUF_Pseudo>(opcode # _RTN_IDXEN) $vdata_in, $vindex, $rsrc, $soffset,
 834                                        (as_i16imm $offset), (as_i1imm $slc))
 835   >;
 836
 837   def : Pat<
 838     (name i32:$vdata_in, v4i32:$rsrc, 0,
 839           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 840           imm:$slc),
 841     (!cast<MUBUF_Pseudo>(opcode # _RTN_OFFEN) $vdata_in, $voffset, $rsrc, $soffset,
 842                                        (as_i16imm $offset), (as_i1imm $slc))
 843   >;
 844
 845   def : Pat<
 846     (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex,
 847           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 848           imm:$slc),
 849     (!cast<MUBUF_Pseudo>(opcode # _RTN_BOTHEN)
 850       $vdata_in,
 851       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
 852       $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc))
 853   >;
 854 }
 855
 856 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_swap, "BUFFER_ATOMIC_SWAP">;
 857 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_add, "BUFFER_ATOMIC_ADD">;
 858 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_sub, "BUFFER_ATOMIC_SUB">;
 859 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smin, "BUFFER_ATOMIC_SMIN">;
 860 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umin, "BUFFER_ATOMIC_UMIN">;
 861 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smax, "BUFFER_ATOMIC_SMAX">;
 862 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umax, "BUFFER_ATOMIC_UMAX">;
 863 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_and, "BUFFER_ATOMIC_AND">;
 864 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_or, "BUFFER_ATOMIC_OR">;
 865 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_xor, "BUFFER_ATOMIC_XOR">;
 866
 867 def : Pat<
 868   (int_amdgcn_buffer_atomic_cmpswap
 869       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
 870       (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 871       imm:$slc),
 872   (EXTRACT_SUBREG
 873     (BUFFER_ATOMIC_CMPSWAP_RTN_OFFSET
 874       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
 875       $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
 876     sub0)
 877 >;
 878
 879 def : Pat<
 880   (int_amdgcn_buffer_atomic_cmpswap
 881       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
 882       (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 883       imm:$slc),
 884   (EXTRACT_SUBREG
 885     (BUFFER_ATOMIC_CMPSWAP_RTN_IDXEN
 886       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
 887       $vindex, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
 888     sub0)
 889 >;
 890
 891 def : Pat<
 892   (int_amdgcn_buffer_atomic_cmpswap
 893       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
 894       (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 895       imm:$slc),
 896   (EXTRACT_SUBREG
 897     (BUFFER_ATOMIC_CMPSWAP_RTN_OFFEN
 898       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
 899       $voffset, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
 900     sub0)
 901 >;
 902
 903 def : Pat<
 904   (int_amdgcn_buffer_atomic_cmpswap
 905       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
 906       (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 907       imm:$slc),
 908   (EXTRACT_SUBREG
 909     (BUFFER_ATOMIC_CMPSWAP_RTN_BOTHEN
 910       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
 911       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
 912       $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
 913     sub0)
 914 >;
 915
 916
 917 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
 918                               PatFrag constant_ld> : Pat <
 919      (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 920                                    i16:$offset, i1:$glc, i1:$slc, i1:$tfe))),
 921      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe)
 922   >;
 923
 924 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
 925                                      ValueType vt, PatFrag atomic_ld> {
 926   def : Pat <
 927      (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 928                                    i16:$offset, i1:$slc))),
 929      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 1, $slc, 0)
 930   >;
 931
 932   def : Pat <
 933     (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))),
 934     (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 1, 0, 0)
 935   >;
 936 }
 937
 938 let Predicates = [isSICI] in {
 939 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
 940 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, az_extloadi8_constant>;
 941 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
 942 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, az_extloadi16_constant>;
 943
 944 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, mubuf_load_atomic>;
 945 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, mubuf_load_atomic>;
 946 } // End Predicates = [isSICI]
 947
 948 multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
 949                                PatFrag ld> {
 950
 951   def : Pat <
 952     (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 953                           i16:$offset, i1:$glc, i1:$slc, i1:$tfe))),
 954     (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe)
 955   >;
 956 }
 957
 958 let Predicates = [Has16BitInsts] in {
 959
 960 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>;
 961 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_constant>;
 962 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, mubuf_sextloadi8>;
 963 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, mubuf_az_extloadi8>;
 964
 965 } // End Predicates = [Has16BitInsts]
 966
 967 class MUBUFScratchLoadPat <MUBUF_Pseudo Instr, ValueType vt, PatFrag ld> : Pat <
 968   (vt (ld (MUBUFScratch v4i32:$srsrc, i32:$vaddr,
 969                         i32:$soffset, u16imm:$offset))),
 970   (Instr $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
 971 >;
 972
 973 def : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, i32, sextloadi8_private>;
 974 def : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, i32, extloadi8_private>;
 975 def : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, i16, sextloadi8_private>;
 976 def : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, i16, extloadi8_private>;
 977 def : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, i32, sextloadi16_private>;
 978 def : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, i32, extloadi16_private>;
 979 def : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, i32, load_private>;
 980 def : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, v2i32, load_private>;
 981 def : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, v4i32, load_private>;
 982
 983 // BUFFER_LOAD_DWORD*, addr64=0
 984 multiclass MUBUF_Load_Dword <ValueType vt,
 985                              MUBUF_Pseudo offset,
 986                              MUBUF_Pseudo offen,
 987                              MUBUF_Pseudo idxen,
 988                              MUBUF_Pseudo bothen> {
 989
 990   def : Pat <
 991     (vt (int_SI_buffer_load_dword v4i32:$rsrc, (i32 imm), i32:$soffset,
 992                                   imm:$offset, 0, 0, imm:$glc, imm:$slc,
 993                                   imm:$tfe)),
 994     (offset $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc),
 995             (as_i1imm $slc), (as_i1imm $tfe))
 996   >;
 997
 998   def : Pat <
 999     (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset,
1000                                   imm:$offset, 1, 0, imm:$glc, imm:$slc,
1001                                   imm:$tfe)),
1002     (offen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc),
1003            (as_i1imm $tfe))
1004   >;
1005
1006   def : Pat <
1007     (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset,
1008                                   imm:$offset, 0, 1, imm:$glc, imm:$slc,
1009                                   imm:$tfe)),
1010     (idxen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc),
1011            (as_i1imm $slc), (as_i1imm $tfe))
1012   >;
1013
1014   def : Pat <
1015     (vt (int_SI_buffer_load_dword v4i32:$rsrc, v2i32:$vaddr, i32:$soffset,
1016                                   imm:$offset, 1, 1, imm:$glc, imm:$slc,
1017                                   imm:$tfe)),
1018     (bothen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc),
1019             (as_i1imm $tfe))
1020   >;
1021 }
1022
1023 defm : MUBUF_Load_Dword <i32, BUFFER_LOAD_DWORD_OFFSET, BUFFER_LOAD_DWORD_OFFEN,
1024                          BUFFER_LOAD_DWORD_IDXEN, BUFFER_LOAD_DWORD_BOTHEN>;
1025 defm : MUBUF_Load_Dword <v2i32, BUFFER_LOAD_DWORDX2_OFFSET, BUFFER_LOAD_DWORDX2_OFFEN,
1026                          BUFFER_LOAD_DWORDX2_IDXEN, BUFFER_LOAD_DWORDX2_BOTHEN>;
1027 defm : MUBUF_Load_Dword <v4i32, BUFFER_LOAD_DWORDX4_OFFSET, BUFFER_LOAD_DWORDX4_OFFEN,
1028                          BUFFER_LOAD_DWORDX4_IDXEN, BUFFER_LOAD_DWORDX4_BOTHEN>;
1029
1030 multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1031                                       ValueType vt, PatFrag atomic_st> {
1032   // Store follows atomic op convention so address is forst
1033   def : Pat <
1034      (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1035                                    i16:$offset, i1:$slc), vt:$val),
1036      (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 1, $slc, 0)
1037   >;
1038
1039   def : Pat <
1040     (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
1041     (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 1, 0, 0)
1042   >;
1043 }
1044 let Predicates = [isSICI] in {
1045 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, global_store_atomic>;
1046 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, global_store_atomic>;
1047 } // End Predicates = [isSICI]
1048
1049
1050 multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1051                                PatFrag st> {
1052
1053   def : Pat <
1054     (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1055                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe)),
1056     (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe)
1057   >;
1058 }
1059
1060 defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>;
1061 defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, global_store>;
1062
1063 class MUBUFScratchStorePat <MUBUF_Pseudo Instr, ValueType vt, PatFrag st> : Pat <
1064   (st vt:$value, (MUBUFScratch v4i32:$srsrc, i32:$vaddr, i32:$soffset,
1065                                u16imm:$offset)),
1066   (Instr $value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
1067 >;
1068
1069 def : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, i32, truncstorei8_private>;
1070 def : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, i32, truncstorei16_private>;
1071 def : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, i16, truncstorei8_private>;
1072 def : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, i16, store_private>;
1073 def : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, i32, store_private>;
1074 def : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, v2i32, store_private>;
1075 def : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, v4i32, store_private>;
1076
1077 //===----------------------------------------------------------------------===//
1078 // MTBUF Patterns
1079 //===----------------------------------------------------------------------===//
1080
1081 // TBUFFER_STORE_FORMAT_*, addr64=0
1082 class MTBUF_StoreResource <ValueType vt, int num_channels, MTBUF_Pseudo opcode> : Pat<
1083   (SItbuffer_store v4i32:$rsrc, vt:$vdata, num_channels, i32:$vaddr,
1084                    i32:$soffset, imm:$inst_offset, imm:$dfmt,
1085                    imm:$nfmt, imm:$offen, imm:$idxen,
1086                    imm:$glc, imm:$slc, imm:$tfe),
1087   (opcode
1088     $vdata, (as_i16imm $inst_offset), (as_i1imm $offen), (as_i1imm $idxen),
1089     (as_i1imm $glc), 0, (as_i8imm $dfmt), (as_i8imm $nfmt), $vaddr, $rsrc,
1090     (as_i1imm $slc), (as_i1imm $tfe), $soffset)
1091 >;
1092
1093 def : MTBUF_StoreResource <i32, 1, TBUFFER_STORE_FORMAT_X>;
1094 def : MTBUF_StoreResource <v2i32, 2, TBUFFER_STORE_FORMAT_XY>;
1095 def : MTBUF_StoreResource <v4i32, 3, TBUFFER_STORE_FORMAT_XYZ>;
1096 def : MTBUF_StoreResource <v4i32, 4, TBUFFER_STORE_FORMAT_XYZW>;
1097
1098 } // End let Predicates = [isGCN]
1099
1100 //===----------------------------------------------------------------------===//
1101 // Target instructions, move to the appropriate target TD file
1102 //===----------------------------------------------------------------------===//
1103
1104 //===----------------------------------------------------------------------===//
1105 // SI
1106 //===----------------------------------------------------------------------===//
1107
1108 class MUBUF_Real_si <bits<7> op, MUBUF_Pseudo ps> :
1109   MUBUF_Real<op, ps>,
1110   Enc64,
1111   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> {
1112   let AssemblerPredicate=isSICI;
1113   let DecoderNamespace="SICI";
1114
1115   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1116   let Inst{12}    = ps.offen;
1117   let Inst{13}    = ps.idxen;
1118   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1119   let Inst{15}    = ps.addr64;
1120   let Inst{16}    = lds;
1121   let Inst{24-18} = op;
1122   let Inst{31-26} = 0x38; //encoding
1123   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1124   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1125   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1126   let Inst{54}    = !if(ps.has_slc, slc, ?);
1127   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1128   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1129 }
1130
1131 multiclass MUBUF_Real_AllAddr_si<bits<7> op> {
1132   def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1133   def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>;
1134   def _OFFEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1135   def _IDXEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1136   def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1137 }
1138
1139 multiclass MUBUF_Real_Atomic_si<bits<7> op> : MUBUF_Real_AllAddr_si<op> {
1140   def _RTN_OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFSET")>;
1141   def _RTN_ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_ADDR64")>;
1142   def _RTN_OFFEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFEN")>;
1143   def _RTN_IDXEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_IDXEN")>;
1144   def _RTN_BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_BOTHEN")>;
1145 }
1146
1147 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_si <0x00>;
1148 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_si <0x01>;
1149 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_si <0x02>;
1150 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_si <0x03>;
1151 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_si <0x04>;
1152 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_si <0x05>;
1153 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_si <0x06>;
1154 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_si <0x07>;
1155 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_si <0x08>;
1156 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_si <0x09>;
1157 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_si <0x0a>;
1158 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_si <0x0b>;
1159 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_si <0x0c>;
1160 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_si <0x0d>;
1161 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_si <0x0e>;
1162 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_si <0x0f>;
1163 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_si <0x18>;
1164 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_si <0x1a>;
1165 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_si <0x1c>;
1166 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_si <0x1d>;
1167 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_si <0x1e>;
1168 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_si <0x1f>;
1169
1170 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_si <0x30>;
1171 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_si <0x31>;
1172 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_si <0x32>;
1173 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_si <0x33>;
1174 //defm BUFFER_ATOMIC_RSUB         : MUBUF_Real_Atomic_si <0x34>;    // isn't on CI & VI
1175 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_si <0x35>;
1176 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_si <0x36>;
1177 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_si <0x37>;
1178 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_si <0x38>;
1179 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_si <0x39>;
1180 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_si <0x3a>;
1181 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_si <0x3b>;
1182 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_si <0x3c>;
1183 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_si <0x3d>;
1184
1185 //defm BUFFER_ATOMIC_FCMPSWAP     : MUBUF_Real_Atomic_si <0x3e>;    // isn't on VI
1186 //defm BUFFER_ATOMIC_FMIN         : MUBUF_Real_Atomic_si <0x3f>;    // isn't on VI
1187 //defm BUFFER_ATOMIC_FMAX         : MUBUF_Real_Atomic_si <0x40>;    // isn't on VI
1188 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_si <0x50>;
1189 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_si <0x51>;
1190 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_si <0x52>;
1191 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_si <0x53>;
1192 //defm BUFFER_ATOMIC_RSUB_X2      : MUBUF_Real_Atomic_si <0x54>;    // isn't on CI & VI
1193 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_si <0x55>;
1194 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_si <0x56>;
1195 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_si <0x57>;
1196 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_si <0x58>;
1197 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_si <0x59>;
1198 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_si <0x5a>;
1199 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_si <0x5b>;
1200 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_si <0x5c>;
1201 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_si <0x5d>;
1202 // FIXME: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on CI.
1203 //defm BUFFER_ATOMIC_FCMPSWAP_X2  : MUBUF_Real_Atomic_si <0x5e">;   // isn't on VI
1204 //defm BUFFER_ATOMIC_FMIN_X2      : MUBUF_Real_Atomic_si <0x5f>;    // isn't on VI
1205 //defm BUFFER_ATOMIC_FMAX_X2      : MUBUF_Real_Atomic_si <0x60>;    // isn't on VI
1206
1207 def BUFFER_WBINVL1_SC_si        : MUBUF_Real_si <0x70, BUFFER_WBINVL1_SC>;
1208 def BUFFER_WBINVL1_si           : MUBUF_Real_si <0x71, BUFFER_WBINVL1>;
1209
1210 class MTBUF_Real_si <bits<3> op, MTBUF_Pseudo ps> :
1211   MTBUF_Real<ps>,
1212   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> {
1213   let AssemblerPredicate=isSICI;
1214   let DecoderNamespace="SICI";
1215
1216   bits<1> addr64;
1217   let Inst{15}    = addr64;
1218   let Inst{18-16} = op;
1219 }
1220
1221 def TBUFFER_LOAD_FORMAT_XYZW_si  : MTBUF_Real_si <3, TBUFFER_LOAD_FORMAT_XYZW>;
1222 def TBUFFER_STORE_FORMAT_X_si    : MTBUF_Real_si <4, TBUFFER_STORE_FORMAT_X>;
1223 def TBUFFER_STORE_FORMAT_XY_si   : MTBUF_Real_si <5, TBUFFER_STORE_FORMAT_XY>;
1224 def TBUFFER_STORE_FORMAT_XYZ_si  : MTBUF_Real_si <6, TBUFFER_STORE_FORMAT_XYZ>;
1225 def TBUFFER_STORE_FORMAT_XYZW_si : MTBUF_Real_si <7, TBUFFER_STORE_FORMAT_XYZW>;
1226
1227
1228 //===----------------------------------------------------------------------===//
1229 // CI
1230 //===----------------------------------------------------------------------===//
1231
1232 class MUBUF_Real_ci <bits<7> op, MUBUF_Pseudo ps> :
1233   MUBUF_Real_si<op, ps> {
1234   let AssemblerPredicate=isCIOnly;
1235   let DecoderNamespace="CI";
1236 }
1237
1238 def BUFFER_WBINVL1_VOL_ci : MUBUF_Real_ci <0x70, BUFFER_WBINVL1_VOL>;
1239
1240
1241 //===----------------------------------------------------------------------===//
1242 // VI
1243 //===----------------------------------------------------------------------===//
1244
1245 class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> :
1246   MUBUF_Real<op, ps>,
1247   Enc64,
1248   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
1249   let AssemblerPredicate=isVI;
1250   let DecoderNamespace="VI";
1251
1252   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1253   let Inst{12}    = ps.offen;
1254   let Inst{13}    = ps.idxen;
1255   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1256   let Inst{16}    = lds;
1257   let Inst{17}    = !if(ps.has_slc, slc, ?);
1258   let Inst{24-18} = op;
1259   let Inst{31-26} = 0x38; //encoding
1260   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1261   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1262   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1263   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1264   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1265 }
1266
1267 multiclass MUBUF_Real_AllAddr_vi<bits<7> op> {
1268   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1269   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1270   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1271   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1272 }
1273
1274 multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
1275   MUBUF_Real_AllAddr_vi<op> {
1276   def _RTN_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFSET")>;
1277   def _RTN_OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFEN")>;
1278   def _RTN_IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_IDXEN")>;
1279   def _RTN_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_BOTHEN")>;
1280 }
1281
1282 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_vi <0x00>;
1283 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_vi <0x01>;
1284 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_vi <0x02>;
1285 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_vi <0x03>;
1286 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_vi <0x04>;
1287 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_vi <0x05>;
1288 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_vi <0x06>;
1289 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_vi <0x07>;
1290 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_vi <0x10>;
1291 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_vi <0x11>;
1292 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_vi <0x12>;
1293 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_vi <0x13>;
1294 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_vi <0x14>;
1295 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_vi <0x15>;
1296 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_vi <0x16>;
1297 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_vi <0x17>;
1298 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_vi <0x18>;
1299 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_vi <0x1a>;
1300 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_vi <0x1c>;
1301 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_vi <0x1d>;
1302 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_vi <0x1e>;
1303 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_vi <0x1f>;
1304
1305 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_vi <0x40>;
1306 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_vi <0x41>;
1307 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_vi <0x42>;
1308 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_vi <0x43>;
1309 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_vi <0x44>;
1310 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_vi <0x45>;
1311 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_vi <0x46>;
1312 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_vi <0x47>;
1313 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_vi <0x48>;
1314 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_vi <0x49>;
1315 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_vi <0x4a>;
1316 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_vi <0x4b>;
1317 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_vi <0x4c>;
1318
1319 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_vi <0x60>;
1320 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_vi <0x61>;
1321 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_vi <0x62>;
1322 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_vi <0x63>;
1323 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_vi <0x64>;
1324 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_vi <0x65>;
1325 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_vi <0x66>;
1326 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_vi <0x67>;
1327 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_vi <0x68>;
1328 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_vi <0x69>;
1329 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_vi <0x6a>;
1330 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_vi <0x6b>;
1331 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_vi <0x6c>;
1332
1333 def BUFFER_WBINVL1_vi           : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>;
1334 def BUFFER_WBINVL1_VOL_vi       : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>;
1335
1336 class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
1337   MTBUF_Real<ps>,
1338   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
1339   let AssemblerPredicate=isVI;
1340   let DecoderNamespace="VI";
1341
1342   let Inst{18-15} = op;
1343 }
1344
1345 def TBUFFER_LOAD_FORMAT_XYZW_vi  : MTBUF_Real_vi <3, TBUFFER_LOAD_FORMAT_XYZW>;
1346 def TBUFFER_STORE_FORMAT_X_vi    : MTBUF_Real_vi <4, TBUFFER_STORE_FORMAT_X>;
1347 def TBUFFER_STORE_FORMAT_XY_vi   : MTBUF_Real_vi <5, TBUFFER_STORE_FORMAT_XY>;
1348 def TBUFFER_STORE_FORMAT_XYZ_vi  : MTBUF_Real_vi <6, TBUFFER_STORE_FORMAT_XYZ>;
1349 def TBUFFER_STORE_FORMAT_XYZW_vi : MTBUF_Real_vi <7, TBUFFER_STORE_FORMAT_XYZW>;
1350