contrib/llvm/lib/Target/AMDGPU/BUFInstructions.td

   1 //===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9
  10 def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">;
  11 def MUBUFAddr64 : ComplexPattern<i64, 7, "SelectMUBUFAddr64">;
  12 def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">;
  13
  14 def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantRoot]>;
  15 def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantRoot], 20>;
  16
  17 def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">;
  18 def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
  19 def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">;
  20 def MUBUFIntrinsicOffset : ComplexPattern<i32, 2, "SelectMUBUFIntrinsicOffset">;
  21 def MUBUFIntrinsicVOffset : ComplexPattern<i32, 3, "SelectMUBUFIntrinsicVOffset">;
  22
  23 class MubufLoad <SDPatternOperator op> : PatFrag <
  24   (ops node:$ptr), (op node:$ptr), [{
  25   auto const AS = cast<MemSDNode>(N)->getAddressSpace();
  26   return AS == AMDGPUASI.GLOBAL_ADDRESS ||
  27          AS == AMDGPUASI.CONSTANT_ADDRESS;
  28 }]>;
  29
  30 def mubuf_load          : MubufLoad <load>;
  31 def mubuf_az_extloadi8  : MubufLoad <az_extloadi8>;
  32 def mubuf_sextloadi8    : MubufLoad <sextloadi8>;
  33 def mubuf_az_extloadi16 : MubufLoad <az_extloadi16>;
  34 def mubuf_sextloadi16   : MubufLoad <sextloadi16>;
  35 def mubuf_load_atomic   : MubufLoad <atomic_load>;
  36
  37 def BUFAddrKind {
  38   int Offset = 0;
  39   int OffEn  = 1;
  40   int IdxEn  = 2;
  41   int BothEn = 3;
  42   int Addr64 = 4;
  43 }
  44
  45 class getAddrName<int addrKind> {
  46   string ret =
  47     !if(!eq(addrKind, BUFAddrKind.Offset), "offset",
  48     !if(!eq(addrKind, BUFAddrKind.OffEn),  "offen",
  49     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "idxen",
  50     !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",
  51     !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",
  52     "")))));
  53 }
  54
  55 class MUBUFAddr64Table <bit is_addr64, string suffix = ""> {
  56   bit IsAddr64 = is_addr64;
  57   string OpName = NAME # suffix;
  58 }
  59
  60 //===----------------------------------------------------------------------===//
  61 // MTBUF classes
  62 //===----------------------------------------------------------------------===//
  63
  64 class MTBUF_Pseudo <string opName, dag outs, dag ins,
  65                     string asmOps, list<dag> pattern=[]> :
  66   InstSI<outs, ins, "", pattern>,
  67   SIMCInstr<opName, SIEncodingFamily.NONE> {
  68
  69   let isPseudo = 1;
  70   let isCodeGenOnly = 1;
  71   let Size = 8;
  72   let UseNamedOperandTable = 1;
  73
  74   string Mnemonic = opName;
  75   string AsmOperands = asmOps;
  76
  77   let VM_CNT = 1;
  78   let EXP_CNT = 1;
  79   let MTBUF = 1;
  80   let Uses = [EXEC];
  81
  82   let hasSideEffects = 0;
  83   let SchedRW = [WriteVMEM];
  84 }
  85
  86 class MTBUF_Real <MTBUF_Pseudo ps> :
  87   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>,
  88   Enc64 {
  89
  90   let isPseudo = 0;
  91   let isCodeGenOnly = 0;
  92
  93   // copy relevant pseudo op flags
  94   let SubtargetPredicate = ps.SubtargetPredicate;
  95   let AsmMatchConverter  = ps.AsmMatchConverter;
  96   let Constraints        = ps.Constraints;
  97   let DisableEncoding    = ps.DisableEncoding;
  98   let TSFlags            = ps.TSFlags;
  99
 100   bits<8> vdata;
 101   bits<12> offset;
 102   bits<1> offen;
 103   bits<1> idxen;
 104   bits<1> glc;
 105   bits<1> addr64;
 106   bits<4> dfmt;
 107   bits<3> nfmt;
 108   bits<8> vaddr;
 109   bits<7> srsrc;
 110   bits<1> slc;
 111   bits<1> tfe;
 112   bits<8> soffset;
 113
 114   let Inst{11-0}  = offset;
 115   let Inst{12}    = offen;
 116   let Inst{13}    = idxen;
 117   let Inst{14}    = glc;
 118   let Inst{22-19} = dfmt;
 119   let Inst{25-23} = nfmt;
 120   let Inst{31-26} = 0x3a; //encoding
 121   let Inst{39-32} = vaddr;
 122   let Inst{47-40} = vdata;
 123   let Inst{52-48} = srsrc{6-2};
 124   let Inst{54}    = slc;
 125   let Inst{55}    = tfe;
 126   let Inst{63-56} = soffset;
 127 }
 128
 129 class MTBUF_Load_Pseudo <string opName, RegisterClass regClass> : MTBUF_Pseudo <
 130   opName, (outs regClass:$dst),
 131   (ins u16imm:$offset, i1imm:$offen, i1imm:$idxen, i1imm:$glc, i1imm:$addr64,
 132        i8imm:$dfmt, i8imm:$nfmt, VGPR_32:$vaddr, SReg_128:$srsrc,
 133        i1imm:$slc, i1imm:$tfe, SCSrc_b32:$soffset),
 134   " $dst, $offset, $offen, $idxen, $glc, $addr64, $dfmt,"#
 135   " $nfmt, $vaddr, $srsrc, $slc, $tfe, $soffset"> {
 136   let mayLoad = 1;
 137   let mayStore = 0;
 138 }
 139
 140 class MTBUF_Store_Pseudo <string opName, RegisterClass regClass> : MTBUF_Pseudo <
 141   opName, (outs),
 142   (ins regClass:$vdata, u16imm:$offset, i1imm:$offen, i1imm:$idxen, i1imm:$glc,
 143        i1imm:$addr64, i8imm:$dfmt, i8imm:$nfmt, VGPR_32:$vaddr,
 144        SReg_128:$srsrc, i1imm:$slc, i1imm:$tfe, SCSrc_b32:$soffset),
 145   " $vdata, $offset, $offen, $idxen, $glc, $addr64, $dfmt,"#
 146   " $nfmt, $vaddr, $srsrc, $slc, $tfe, $soffset"> {
 147   let mayLoad = 0;
 148   let mayStore = 1;
 149 }
 150
 151 //===----------------------------------------------------------------------===//
 152 // MUBUF classes
 153 //===----------------------------------------------------------------------===//
 154
 155 class MUBUF_Pseudo <string opName, dag outs, dag ins,
 156                     string asmOps, list<dag> pattern=[]> :
 157   InstSI<outs, ins, "", pattern>,
 158   SIMCInstr<opName, SIEncodingFamily.NONE> {
 159
 160   let isPseudo = 1;
 161   let isCodeGenOnly = 1;
 162   let Size = 8;
 163   let UseNamedOperandTable = 1;
 164
 165   string Mnemonic = opName;
 166   string AsmOperands = asmOps;
 167
 168   let VM_CNT = 1;
 169   let EXP_CNT = 1;
 170   let MUBUF = 1;
 171   let Uses = [EXEC];
 172   let hasSideEffects = 0;
 173   let SchedRW = [WriteVMEM];
 174
 175   let AsmMatchConverter = "cvtMubuf";
 176
 177   bits<1> offen       = 0;
 178   bits<1> idxen       = 0;
 179   bits<1> addr64      = 0;
 180   bits<1> has_vdata   = 1;
 181   bits<1> has_vaddr   = 1;
 182   bits<1> has_glc     = 1;
 183   bits<1> glc_value   = 0; // the value for glc if no such operand
 184   bits<1> has_srsrc   = 1;
 185   bits<1> has_soffset = 1;
 186   bits<1> has_offset  = 1;
 187   bits<1> has_slc     = 1;
 188   bits<1> has_tfe     = 1;
 189 }
 190
 191 class MUBUF_Real <bits<7> op, MUBUF_Pseudo ps> :
 192   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 193
 194   let isPseudo = 0;
 195   let isCodeGenOnly = 0;
 196
 197   // copy relevant pseudo op flags
 198   let SubtargetPredicate = ps.SubtargetPredicate;
 199   let AsmMatchConverter  = ps.AsmMatchConverter;
 200   let Constraints        = ps.Constraints;
 201   let DisableEncoding    = ps.DisableEncoding;
 202   let TSFlags            = ps.TSFlags;
 203
 204   bits<12> offset;
 205   bits<1>  glc;
 206   bits<1>  lds = 0;
 207   bits<8>  vaddr;
 208   bits<8>  vdata;
 209   bits<7>  srsrc;
 210   bits<1>  slc;
 211   bits<1>  tfe;
 212   bits<8>  soffset;
 213 }
 214
 215
 216 // For cache invalidation instructions.
 217 class MUBUF_Invalidate <string opName, SDPatternOperator node> :
 218   MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {
 219
 220   let AsmMatchConverter = "";
 221
 222   let hasSideEffects = 1;
 223   let mayStore = 1;
 224
 225   // Set everything to 0.
 226   let offen       = 0;
 227   let idxen       = 0;
 228   let addr64      = 0;
 229   let has_vdata   = 0;
 230   let has_vaddr   = 0;
 231   let has_glc     = 0;
 232   let glc_value   = 0;
 233   let has_srsrc   = 0;
 234   let has_soffset = 0;
 235   let has_offset  = 0;
 236   let has_slc     = 0;
 237   let has_tfe     = 0;
 238 }
 239
 240 class getMUBUFInsDA<list<RegisterClass> vdataList,
 241                     list<RegisterClass> vaddrList=[]> {
 242   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 243   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 244   dag InsNoData = !if(!empty(vaddrList),
 245     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 246          offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe),
 247     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 248          offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe)
 249   );
 250   dag InsData = !if(!empty(vaddrList),
 251     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 252          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe),
 253     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 254          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe)
 255   );
 256   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 257 }
 258
 259 class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
 260   dag ret =
 261     !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList>.ret,
 262     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMUBUFInsDA<vdataList, [VGPR_32]>.ret,
 263     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMUBUFInsDA<vdataList, [VGPR_32]>.ret,
 264     !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64]>.ret,
 265     !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64]>.ret,
 266     (ins))))));
 267 }
 268
 269 class getMUBUFAsmOps<int addrKind> {
 270   string Pfx =
 271     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",
 272     !if(!eq(addrKind, BUFAddrKind.OffEn),  "$vaddr, $srsrc, $soffset offen",
 273     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "$vaddr, $srsrc, $soffset idxen",
 274     !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",
 275     !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",
 276     "")))));
 277   string ret = Pfx # "$offset";
 278 }
 279
 280 class MUBUF_SetupAddr<int addrKind> {
 281   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 282                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 283
 284   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 285                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 286
 287   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 288
 289   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 290 }
 291
 292 class MUBUF_Load_Pseudo <string opName,
 293                          int addrKind,
 294                          RegisterClass vdataClass,
 295                          list<dag> pattern=[],
 296                          // Workaround bug bz30254
 297                          int addrKindCopy = addrKind>
 298   : MUBUF_Pseudo<opName,
 299                  (outs vdataClass:$vdata),
 300                  getMUBUFIns<addrKindCopy>.ret,
 301                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
 302                  pattern>,
 303     MUBUF_SetupAddr<addrKindCopy> {
 304   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 305   let mayLoad = 1;
 306   let mayStore = 0;
 307 }
 308
 309 // FIXME: tfe can't be an operand because it requires a separate
 310 // opcode because it needs an N+1 register class dest register.
 311 multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 312                               ValueType load_vt = i32,
 313                               SDPatternOperator ld = null_frag> {
 314
 315   def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 316     [(set load_vt:$vdata,
 317      (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>,
 318     MUBUFAddr64Table<0>;
 319
 320   def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 321     [(set load_vt:$vdata,
 322      (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>,
 323     MUBUFAddr64Table<1>;
 324
 325   def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 326   def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 327   def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 328
 329   let DisableWQM = 1 in {
 330     def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 331     def _OFFEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 332     def _IDXEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 333     def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 334   }
 335 }
 336
 337 class MUBUF_Store_Pseudo <string opName,
 338                           int addrKind,
 339                           RegisterClass vdataClass,
 340                           list<dag> pattern=[],
 341                           // Workaround bug bz30254
 342                           int addrKindCopy = addrKind,
 343                           RegisterClass vdataClassCopy = vdataClass>
 344   : MUBUF_Pseudo<opName,
 345                  (outs),
 346                  getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 347                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
 348                  pattern>,
 349     MUBUF_SetupAddr<addrKindCopy> {
 350   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 351   let mayLoad = 0;
 352   let mayStore = 1;
 353 }
 354
 355 multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 356                                ValueType store_vt = i32,
 357                                SDPatternOperator st = null_frag> {
 358
 359   def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 360     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 361                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>,
 362     MUBUFAddr64Table<0>;
 363
 364   def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 365     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 366                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>,
 367     MUBUFAddr64Table<1>;
 368
 369   def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 370   def _IDXEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 371   def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 372
 373   let DisableWQM = 1 in {
 374     def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 375     def _OFFEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 376     def _IDXEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 377     def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 378   }
 379 }
 380
 381
 382 class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
 383                           list<RegisterClass> vaddrList=[]> {
 384   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 385   dag ret = !if(vdata_in,
 386     !if(!empty(vaddrList),
 387       (ins vdataClass:$vdata_in,
 388            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc),
 389       (ins vdataClass:$vdata_in, vaddrClass:$vaddr,
 390            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc)
 391     ),
 392     !if(!empty(vaddrList),
 393       (ins vdataClass:$vdata,
 394            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc),
 395       (ins vdataClass:$vdata, vaddrClass:$vaddr,
 396            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc)
 397   ));
 398 }
 399
 400 class getMUBUFAtomicIns<int addrKind,
 401                         RegisterClass vdataClass,
 402                         bit vdata_in,
 403                         // Workaround bug bz30254
 404                         RegisterClass vdataClassCopy=vdataClass> {
 405   dag ret =
 406     !if(!eq(addrKind, BUFAddrKind.Offset),
 407             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret,
 408     !if(!eq(addrKind, BUFAddrKind.OffEn),
 409             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 410     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 411             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 412     !if(!eq(addrKind, BUFAddrKind.BothEn),
 413             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 414     !if(!eq(addrKind, BUFAddrKind.Addr64),
 415             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 416     (ins))))));
 417 }
 418
 419 class MUBUF_Atomic_Pseudo<string opName,
 420                           int addrKind,
 421                           dag outs,
 422                           dag ins,
 423                           string asmOps,
 424                           list<dag> pattern=[],
 425                           // Workaround bug bz30254
 426                           int addrKindCopy = addrKind>
 427   : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,
 428     MUBUF_SetupAddr<addrKindCopy> {
 429   let mayStore = 1;
 430   let mayLoad = 1;
 431   let hasPostISelHook = 1;
 432   let hasSideEffects = 1;
 433   let DisableWQM = 1;
 434   let has_glc = 0;
 435   let has_tfe = 0;
 436 }
 437
 438 class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
 439                                RegisterClass vdataClass,
 440                                list<dag> pattern=[],
 441                                // Workaround bug bz30254
 442                                int addrKindCopy = addrKind,
 443                                RegisterClass vdataClassCopy = vdataClass>
 444   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 445                         (outs),
 446                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret,
 447                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc",
 448                         pattern>,
 449     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> {
 450   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 451   let glc_value = 0;
 452   let AsmMatchConverter = "cvtMubufAtomic";
 453 }
 454
 455 class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
 456                              RegisterClass vdataClass,
 457                              list<dag> pattern=[],
 458                              // Workaround bug bz30254
 459                              int addrKindCopy = addrKind,
 460                              RegisterClass vdataClassCopy = vdataClass>
 461   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 462                         (outs vdataClassCopy:$vdata),
 463                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret,
 464                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc",
 465                         pattern>,
 466     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> {
 467   let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret;
 468   let glc_value = 1;
 469   let Constraints = "$vdata = $vdata_in";
 470   let DisableEncoding = "$vdata_in";
 471   let AsmMatchConverter = "cvtMubufAtomicReturn";
 472 }
 473
 474 multiclass MUBUF_Pseudo_Atomics <string opName,
 475                                  RegisterClass vdataClass,
 476                                  ValueType vdataType,
 477                                  SDPatternOperator atomic> {
 478
 479   def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>,
 480                 MUBUFAddr64Table <0>;
 481   def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>,
 482                 MUBUFAddr64Table <1>;
 483   def _OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 484   def _IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 485   def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 486
 487   def _RTN_OFFSET : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 488     [(set vdataType:$vdata,
 489      (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc),
 490              vdataType:$vdata_in))]>,
 491     MUBUFAddr64Table <0, "_RTN">;
 492
 493   def _RTN_ADDR64 : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 494     [(set vdataType:$vdata,
 495      (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc),
 496              vdataType:$vdata_in))]>,
 497     MUBUFAddr64Table <1, "_RTN">;
 498
 499   def _RTN_OFFEN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 500   def _RTN_IDXEN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 501   def _RTN_BOTHEN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 502 }
 503
 504
 505 //===----------------------------------------------------------------------===//
 506 // MUBUF Instructions
 507 //===----------------------------------------------------------------------===//
 508
 509 let SubtargetPredicate = isGCN in {
 510
 511 defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads <
 512   "buffer_load_format_x", VGPR_32
 513 >;
 514 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
 515   "buffer_load_format_xy", VReg_64
 516 >;
 517 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <
 518   "buffer_load_format_xyz", VReg_96
 519 >;
 520 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <
 521   "buffer_load_format_xyzw", VReg_128
 522 >;
 523 defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <
 524   "buffer_store_format_x", VGPR_32
 525 >;
 526 defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <
 527   "buffer_store_format_xy", VReg_64
 528 >;
 529 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <
 530   "buffer_store_format_xyz", VReg_96
 531 >;
 532 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <
 533   "buffer_store_format_xyzw", VReg_128
 534 >;
 535 defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads <
 536   "buffer_load_ubyte", VGPR_32, i32, mubuf_az_extloadi8
 537 >;
 538 defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads <
 539   "buffer_load_sbyte", VGPR_32, i32, mubuf_sextloadi8
 540 >;
 541 defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads <
 542   "buffer_load_ushort", VGPR_32, i32, mubuf_az_extloadi16
 543 >;
 544 defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads <
 545   "buffer_load_sshort", VGPR_32, i32, mubuf_sextloadi16
 546 >;
 547 defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads <
 548   "buffer_load_dword", VGPR_32, i32, mubuf_load
 549 >;
 550 defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
 551   "buffer_load_dwordx2", VReg_64, v2i32, mubuf_load
 552 >;
 553 defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
 554   "buffer_load_dwordx3", VReg_96, untyped, mubuf_load
 555 >;
 556 defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
 557   "buffer_load_dwordx4", VReg_128, v4i32, mubuf_load
 558 >;
 559 defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
 560   "buffer_store_byte", VGPR_32, i32, truncstorei8_global
 561 >;
 562 defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
 563   "buffer_store_short", VGPR_32, i32, truncstorei16_global
 564 >;
 565 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
 566   "buffer_store_dword", VGPR_32, i32, global_store
 567 >;
 568 defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
 569   "buffer_store_dwordx2", VReg_64, v2i32, global_store
 570 >;
 571 defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
 572   "buffer_store_dwordx3", VReg_96, untyped, global_store
 573 >;
 574 defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
 575   "buffer_store_dwordx4", VReg_128, v4i32, global_store
 576 >;
 577 defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
 578   "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global
 579 >;
 580 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <
 581   "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag
 582 >;
 583 defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <
 584   "buffer_atomic_add", VGPR_32, i32, atomic_add_global
 585 >;
 586 defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <
 587   "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global
 588 >;
 589 defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <
 590   "buffer_atomic_smin", VGPR_32, i32, atomic_min_global
 591 >;
 592 defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <
 593   "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global
 594 >;
 595 defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <
 596   "buffer_atomic_smax", VGPR_32, i32, atomic_max_global
 597 >;
 598 defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <
 599   "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global
 600 >;
 601 defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <
 602   "buffer_atomic_and", VGPR_32, i32, atomic_and_global
 603 >;
 604 defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <
 605   "buffer_atomic_or", VGPR_32, i32, atomic_or_global
 606 >;
 607 defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <
 608   "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global
 609 >;
 610 defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <
 611   "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global
 612 >;
 613 defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <
 614   "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global
 615 >;
 616 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <
 617   "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global
 618 >;
 619 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <
 620   "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag
 621 >;
 622 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <
 623   "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global
 624 >;
 625 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <
 626   "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global
 627 >;
 628 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <
 629   "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global
 630 >;
 631 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <
 632   "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global
 633 >;
 634 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <
 635   "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global
 636 >;
 637 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <
 638   "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global
 639 >;
 640 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <
 641   "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global
 642 >;
 643 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <
 644   "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global
 645 >;
 646 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <
 647   "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global
 648 >;
 649 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <
 650   "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global
 651 >;
 652 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
 653   "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global
 654 >;
 655
 656 let SubtargetPredicate = isSI in { // isn't on CI & VI
 657 /*
 658 defm BUFFER_ATOMIC_RSUB        : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
 659 defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">;
 660 defm BUFFER_ATOMIC_FMIN        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">;
 661 defm BUFFER_ATOMIC_FMAX        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">;
 662 defm BUFFER_ATOMIC_RSUB_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
 663 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">;
 664 defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">;
 665 defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">;
 666 */
 667
 668 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
 669                                           int_amdgcn_buffer_wbinvl1_sc>;
 670 }
 671
 672 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
 673                                        int_amdgcn_buffer_wbinvl1>;
 674
 675 //===----------------------------------------------------------------------===//
 676 // MTBUF Instructions
 677 //===----------------------------------------------------------------------===//
 678
 679 //def TBUFFER_LOAD_FORMAT_X    : MTBUF_ <0, "tbuffer_load_format_x", []>;
 680 //def TBUFFER_LOAD_FORMAT_XY   : MTBUF_ <1, "tbuffer_load_format_xy", []>;
 681 //def TBUFFER_LOAD_FORMAT_XYZ  : MTBUF_ <2, "tbuffer_load_format_xyz", []>;
 682 def TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Load_Pseudo  <"tbuffer_load_format_xyzw", VReg_128>;
 683 def TBUFFER_STORE_FORMAT_X    : MTBUF_Store_Pseudo <"tbuffer_store_format_x", VGPR_32>;
 684 def TBUFFER_STORE_FORMAT_XY   : MTBUF_Store_Pseudo <"tbuffer_store_format_xy", VReg_64>;
 685 def TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Store_Pseudo <"tbuffer_store_format_xyz", VReg_128>;
 686 def TBUFFER_STORE_FORMAT_XYZW : MTBUF_Store_Pseudo <"tbuffer_store_format_xyzw", VReg_128>;
 687
 688 } // End let SubtargetPredicate = isGCN
 689
 690 let SubtargetPredicate = isCIVI in {
 691
 692 //===----------------------------------------------------------------------===//
 693 // Instruction definitions for CI and newer.
 694 //===----------------------------------------------------------------------===//
 695 // Remaining instructions:
 696 // BUFFER_LOAD_DWORDX3
 697 // BUFFER_STORE_DWORDX3
 698
 699 def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
 700                                            int_amdgcn_buffer_wbinvl1_vol>;
 701
 702 } // End let SubtargetPredicate = isCIVI
 703
 704 //===----------------------------------------------------------------------===//
 705 // MUBUF Patterns
 706 //===----------------------------------------------------------------------===//
 707
 708 let Predicates = [isGCN] in {
 709
 710 // Offset in an 32-bit VGPR
 711 def : Pat <
 712   (SIload_constant v4i32:$sbase, i32:$voff),
 713   (BUFFER_LOAD_DWORD_OFFEN $voff, $sbase, (i32 0), 0, 0, 0, 0)
 714 >;
 715
 716
 717 //===----------------------------------------------------------------------===//
 718 // buffer_load/store_format patterns
 719 //===----------------------------------------------------------------------===//
 720
 721 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
 722                                   string opcode> {
 723   def : Pat<
 724     (vt (name v4i32:$rsrc, 0,
 725               (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 726               imm:$glc, imm:$slc)),
 727     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
 728       (as_i1imm $glc), (as_i1imm $slc), 0)
 729   >;
 730
 731   def : Pat<
 732     (vt (name v4i32:$rsrc, i32:$vindex,
 733               (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 734               imm:$glc, imm:$slc)),
 735     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
 736       (as_i1imm $glc), (as_i1imm $slc), 0)
 737   >;
 738
 739   def : Pat<
 740     (vt (name v4i32:$rsrc, 0,
 741               (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 742               imm:$glc, imm:$slc)),
 743     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
 744       (as_i1imm $glc), (as_i1imm $slc), 0)
 745   >;
 746
 747   def : Pat<
 748     (vt (name v4i32:$rsrc, i32:$vindex,
 749               (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 750               imm:$glc, imm:$slc)),
 751     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
 752       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
 753       $rsrc, $soffset, (as_i16imm $offset),
 754       (as_i1imm $glc), (as_i1imm $slc), 0)
 755   >;
 756 }
 757
 758 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
 759 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
 760 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;
 761 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">;
 762 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">;
 763 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">;
 764
 765 multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
 766                                    string opcode> {
 767   def : Pat<
 768     (name vt:$vdata, v4i32:$rsrc, 0,
 769           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 770           imm:$glc, imm:$slc),
 771     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset),
 772                                     (as_i1imm $glc), (as_i1imm $slc), 0)
 773   >;
 774
 775   def : Pat<
 776     (name vt:$vdata, v4i32:$rsrc, i32:$vindex,
 777           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 778           imm:$glc, imm:$slc),
 779     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
 780                                    (as_i16imm $offset), (as_i1imm $glc),
 781                                    (as_i1imm $slc), 0)
 782   >;
 783
 784   def : Pat<
 785     (name vt:$vdata, v4i32:$rsrc, 0,
 786           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 787           imm:$glc, imm:$slc),
 788     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
 789                                    (as_i16imm $offset), (as_i1imm $glc),
 790                                    (as_i1imm $slc), 0)
 791   >;
 792
 793   def : Pat<
 794     (name vt:$vdata, v4i32:$rsrc, i32:$vindex,
 795           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 796           imm:$glc, imm:$slc),
 797     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
 798       $vdata,
 799       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
 800       $rsrc, $soffset, (as_i16imm $offset),
 801       (as_i1imm $glc), (as_i1imm $slc), 0)
 802   >;
 803 }
 804
 805 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
 806 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
 807 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
 808 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, f32, "BUFFER_STORE_DWORD">;
 809 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v2f32, "BUFFER_STORE_DWORDX2">;
 810 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v4f32, "BUFFER_STORE_DWORDX4">;
 811
 812 //===----------------------------------------------------------------------===//
 813 // buffer_atomic patterns
 814 //===----------------------------------------------------------------------===//
 815
 816 multiclass BufferAtomicPatterns<SDPatternOperator name, string opcode> {
 817   def : Pat<
 818     (name i32:$vdata_in, v4i32:$rsrc, 0,
 819           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 820           imm:$slc),
 821     (!cast<MUBUF_Pseudo>(opcode # _RTN_OFFSET) $vdata_in, $rsrc, $soffset,
 822                                         (as_i16imm $offset), (as_i1imm $slc))
 823   >;
 824
 825   def : Pat<
 826     (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex,
 827           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 828           imm:$slc),
 829     (!cast<MUBUF_Pseudo>(opcode # _RTN_IDXEN) $vdata_in, $vindex, $rsrc, $soffset,
 830                                        (as_i16imm $offset), (as_i1imm $slc))
 831   >;
 832
 833   def : Pat<
 834     (name i32:$vdata_in, v4i32:$rsrc, 0,
 835           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 836           imm:$slc),
 837     (!cast<MUBUF_Pseudo>(opcode # _RTN_OFFEN) $vdata_in, $voffset, $rsrc, $soffset,
 838                                        (as_i16imm $offset), (as_i1imm $slc))
 839   >;
 840
 841   def : Pat<
 842     (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex,
 843           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 844           imm:$slc),
 845     (!cast<MUBUF_Pseudo>(opcode # _RTN_BOTHEN)
 846       $vdata_in,
 847       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
 848       $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc))
 849   >;
 850 }
 851
 852 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_swap, "BUFFER_ATOMIC_SWAP">;
 853 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_add, "BUFFER_ATOMIC_ADD">;
 854 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_sub, "BUFFER_ATOMIC_SUB">;
 855 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smin, "BUFFER_ATOMIC_SMIN">;
 856 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umin, "BUFFER_ATOMIC_UMIN">;
 857 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smax, "BUFFER_ATOMIC_SMAX">;
 858 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umax, "BUFFER_ATOMIC_UMAX">;
 859 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_and, "BUFFER_ATOMIC_AND">;
 860 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_or, "BUFFER_ATOMIC_OR">;
 861 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_xor, "BUFFER_ATOMIC_XOR">;
 862
 863 def : Pat<
 864   (int_amdgcn_buffer_atomic_cmpswap
 865       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
 866       (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 867       imm:$slc),
 868   (EXTRACT_SUBREG
 869     (BUFFER_ATOMIC_CMPSWAP_RTN_OFFSET
 870       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
 871       $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
 872     sub0)
 873 >;
 874
 875 def : Pat<
 876   (int_amdgcn_buffer_atomic_cmpswap
 877       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
 878       (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 879       imm:$slc),
 880   (EXTRACT_SUBREG
 881     (BUFFER_ATOMIC_CMPSWAP_RTN_IDXEN
 882       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
 883       $vindex, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
 884     sub0)
 885 >;
 886
 887 def : Pat<
 888   (int_amdgcn_buffer_atomic_cmpswap
 889       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
 890       (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 891       imm:$slc),
 892   (EXTRACT_SUBREG
 893     (BUFFER_ATOMIC_CMPSWAP_RTN_OFFEN
 894       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
 895       $voffset, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
 896     sub0)
 897 >;
 898
 899 def : Pat<
 900   (int_amdgcn_buffer_atomic_cmpswap
 901       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
 902       (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 903       imm:$slc),
 904   (EXTRACT_SUBREG
 905     (BUFFER_ATOMIC_CMPSWAP_RTN_BOTHEN
 906       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
 907       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
 908       $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
 909     sub0)
 910 >;
 911
 912
 913 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
 914                               PatFrag constant_ld> : Pat <
 915      (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 916                                    i16:$offset, i1:$glc, i1:$slc, i1:$tfe))),
 917      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe)
 918   >;
 919
 920 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
 921                                      ValueType vt, PatFrag atomic_ld> {
 922   def : Pat <
 923      (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 924                                    i16:$offset, i1:$slc))),
 925      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 1, $slc, 0)
 926   >;
 927
 928   def : Pat <
 929     (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))),
 930     (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 1, 0, 0)
 931   >;
 932 }
 933
 934 let Predicates = [isSICI] in {
 935 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
 936 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, az_extloadi8_constant>;
 937 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
 938 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, az_extloadi16_constant>;
 939
 940 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, mubuf_load_atomic>;
 941 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, mubuf_load_atomic>;
 942 } // End Predicates = [isSICI]
 943
 944 multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
 945                                PatFrag ld> {
 946
 947   def : Pat <
 948     (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 949                           i16:$offset, i1:$glc, i1:$slc, i1:$tfe))),
 950     (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe)
 951   >;
 952 }
 953
 954 let Predicates = [Has16BitInsts] in {
 955
 956 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>;
 957 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_constant>;
 958 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, mubuf_sextloadi8>;
 959 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, mubuf_az_extloadi8>;
 960
 961 } // End Predicates = [Has16BitInsts]
 962
 963 multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen,
 964                                 MUBUF_Pseudo InstrOffset,
 965                                 ValueType vt, PatFrag ld> {
 966   def : Pat <
 967     (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
 968                                i32:$soffset, u16imm:$offset))),
 969     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
 970   >;
 971
 972   def : Pat <
 973     (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))),
 974     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0)
 975   >;
 976 }
 977
 978 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>;
 979 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>;
 980 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>;
 981 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>;
 982 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>;
 983 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>;
 984 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>;
 985 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>;
 986 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>;
 987
 988 // BUFFER_LOAD_DWORD*, addr64=0
 989 multiclass MUBUF_Load_Dword <ValueType vt,
 990                              MUBUF_Pseudo offset,
 991                              MUBUF_Pseudo offen,
 992                              MUBUF_Pseudo idxen,
 993                              MUBUF_Pseudo bothen> {
 994
 995   def : Pat <
 996     (vt (int_SI_buffer_load_dword v4i32:$rsrc, (i32 imm), i32:$soffset,
 997                                   imm:$offset, 0, 0, imm:$glc, imm:$slc,
 998                                   imm:$tfe)),
 999     (offset $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc),
1000             (as_i1imm $slc), (as_i1imm $tfe))
1001   >;
1002
1003   def : Pat <
1004     (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset,
1005                                   imm:$offset, 1, 0, imm:$glc, imm:$slc,
1006                                   imm:$tfe)),
1007     (offen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc),
1008            (as_i1imm $tfe))
1009   >;
1010
1011   def : Pat <
1012     (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset,
1013                                   imm:$offset, 0, 1, imm:$glc, imm:$slc,
1014                                   imm:$tfe)),
1015     (idxen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc),
1016            (as_i1imm $slc), (as_i1imm $tfe))
1017   >;
1018
1019   def : Pat <
1020     (vt (int_SI_buffer_load_dword v4i32:$rsrc, v2i32:$vaddr, i32:$soffset,
1021                                   imm:$offset, 1, 1, imm:$glc, imm:$slc,
1022                                   imm:$tfe)),
1023     (bothen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc),
1024             (as_i1imm $tfe))
1025   >;
1026 }
1027
1028 defm : MUBUF_Load_Dword <i32, BUFFER_LOAD_DWORD_OFFSET, BUFFER_LOAD_DWORD_OFFEN,
1029                          BUFFER_LOAD_DWORD_IDXEN, BUFFER_LOAD_DWORD_BOTHEN>;
1030 defm : MUBUF_Load_Dword <v2i32, BUFFER_LOAD_DWORDX2_OFFSET, BUFFER_LOAD_DWORDX2_OFFEN,
1031                          BUFFER_LOAD_DWORDX2_IDXEN, BUFFER_LOAD_DWORDX2_BOTHEN>;
1032 defm : MUBUF_Load_Dword <v4i32, BUFFER_LOAD_DWORDX4_OFFSET, BUFFER_LOAD_DWORDX4_OFFEN,
1033                          BUFFER_LOAD_DWORDX4_IDXEN, BUFFER_LOAD_DWORDX4_BOTHEN>;
1034
1035 multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1036                                       ValueType vt, PatFrag atomic_st> {
1037   // Store follows atomic op convention so address is forst
1038   def : Pat <
1039      (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1040                                    i16:$offset, i1:$slc), vt:$val),
1041      (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 1, $slc, 0)
1042   >;
1043
1044   def : Pat <
1045     (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
1046     (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 1, 0, 0)
1047   >;
1048 }
1049 let Predicates = [isSICI] in {
1050 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, global_store_atomic>;
1051 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, global_store_atomic>;
1052 } // End Predicates = [isSICI]
1053
1054
1055 multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1056                                PatFrag st> {
1057
1058   def : Pat <
1059     (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1060                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe)),
1061     (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe)
1062   >;
1063 }
1064
1065 defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>;
1066 defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, global_store>;
1067
1068 multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen,
1069                                  MUBUF_Pseudo InstrOffset,
1070                                  ValueType vt, PatFrag st> {
1071   def : Pat <
1072     (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1073                                       i32:$soffset, u16imm:$offset)),
1074     (InstrOffen $value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
1075   >;
1076
1077   def : Pat <
1078     (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,
1079                                        u16imm:$offset)),
1080     (InstrOffset $value, $srsrc, $soffset, $offset, 0, 0, 0)
1081   >;
1082 }
1083
1084 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>;
1085 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>;
1086 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>;
1087 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>;
1088 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>;
1089 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private>;
1090 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private>;
1091
1092 //===----------------------------------------------------------------------===//
1093 // MTBUF Patterns
1094 //===----------------------------------------------------------------------===//
1095
1096 // TBUFFER_STORE_FORMAT_*, addr64=0
1097 class MTBUF_StoreResource <ValueType vt, int num_channels, MTBUF_Pseudo opcode> : Pat<
1098   (SItbuffer_store v4i32:$rsrc, vt:$vdata, num_channels, i32:$vaddr,
1099                    i32:$soffset, imm:$inst_offset, imm:$dfmt,
1100                    imm:$nfmt, imm:$offen, imm:$idxen,
1101                    imm:$glc, imm:$slc, imm:$tfe),
1102   (opcode
1103     $vdata, (as_i16imm $inst_offset), (as_i1imm $offen), (as_i1imm $idxen),
1104     (as_i1imm $glc), 0, (as_i8imm $dfmt), (as_i8imm $nfmt), $vaddr, $rsrc,
1105     (as_i1imm $slc), (as_i1imm $tfe), $soffset)
1106 >;
1107
1108 def : MTBUF_StoreResource <i32, 1, TBUFFER_STORE_FORMAT_X>;
1109 def : MTBUF_StoreResource <v2i32, 2, TBUFFER_STORE_FORMAT_XY>;
1110 def : MTBUF_StoreResource <v4i32, 3, TBUFFER_STORE_FORMAT_XYZ>;
1111 def : MTBUF_StoreResource <v4i32, 4, TBUFFER_STORE_FORMAT_XYZW>;
1112
1113 } // End let Predicates = [isGCN]
1114
1115 //===----------------------------------------------------------------------===//
1116 // Target instructions, move to the appropriate target TD file
1117 //===----------------------------------------------------------------------===//
1118
1119 //===----------------------------------------------------------------------===//
1120 // SI
1121 //===----------------------------------------------------------------------===//
1122
1123 class MUBUF_Real_si <bits<7> op, MUBUF_Pseudo ps> :
1124   MUBUF_Real<op, ps>,
1125   Enc64,
1126   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> {
1127   let AssemblerPredicate=isSICI;
1128   let DecoderNamespace="SICI";
1129
1130   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1131   let Inst{12}    = ps.offen;
1132   let Inst{13}    = ps.idxen;
1133   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1134   let Inst{15}    = ps.addr64;
1135   let Inst{16}    = lds;
1136   let Inst{24-18} = op;
1137   let Inst{31-26} = 0x38; //encoding
1138   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1139   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1140   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1141   let Inst{54}    = !if(ps.has_slc, slc, ?);
1142   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1143   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1144 }
1145
1146 multiclass MUBUF_Real_AllAddr_si<bits<7> op> {
1147   def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1148   def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>;
1149   def _OFFEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1150   def _IDXEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1151   def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1152 }
1153
1154 multiclass MUBUF_Real_Atomic_si<bits<7> op> : MUBUF_Real_AllAddr_si<op> {
1155   def _RTN_OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFSET")>;
1156   def _RTN_ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_ADDR64")>;
1157   def _RTN_OFFEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFEN")>;
1158   def _RTN_IDXEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_IDXEN")>;
1159   def _RTN_BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_BOTHEN")>;
1160 }
1161
1162 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_si <0x00>;
1163 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_si <0x01>;
1164 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_si <0x02>;
1165 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_si <0x03>;
1166 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_si <0x04>;
1167 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_si <0x05>;
1168 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_si <0x06>;
1169 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_si <0x07>;
1170 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_si <0x08>;
1171 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_si <0x09>;
1172 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_si <0x0a>;
1173 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_si <0x0b>;
1174 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_si <0x0c>;
1175 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_si <0x0d>;
1176 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_si <0x0e>;
1177 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_si <0x0f>;
1178 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_si <0x18>;
1179 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_si <0x1a>;
1180 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_si <0x1c>;
1181 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_si <0x1d>;
1182 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_si <0x1e>;
1183 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_si <0x1f>;
1184
1185 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_si <0x30>;
1186 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_si <0x31>;
1187 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_si <0x32>;
1188 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_si <0x33>;
1189 //defm BUFFER_ATOMIC_RSUB         : MUBUF_Real_Atomic_si <0x34>;    // isn't on CI & VI
1190 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_si <0x35>;
1191 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_si <0x36>;
1192 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_si <0x37>;
1193 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_si <0x38>;
1194 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_si <0x39>;
1195 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_si <0x3a>;
1196 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_si <0x3b>;
1197 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_si <0x3c>;
1198 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_si <0x3d>;
1199
1200 //defm BUFFER_ATOMIC_FCMPSWAP     : MUBUF_Real_Atomic_si <0x3e>;    // isn't on VI
1201 //defm BUFFER_ATOMIC_FMIN         : MUBUF_Real_Atomic_si <0x3f>;    // isn't on VI
1202 //defm BUFFER_ATOMIC_FMAX         : MUBUF_Real_Atomic_si <0x40>;    // isn't on VI
1203 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_si <0x50>;
1204 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_si <0x51>;
1205 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_si <0x52>;
1206 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_si <0x53>;
1207 //defm BUFFER_ATOMIC_RSUB_X2      : MUBUF_Real_Atomic_si <0x54>;    // isn't on CI & VI
1208 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_si <0x55>;
1209 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_si <0x56>;
1210 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_si <0x57>;
1211 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_si <0x58>;
1212 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_si <0x59>;
1213 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_si <0x5a>;
1214 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_si <0x5b>;
1215 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_si <0x5c>;
1216 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_si <0x5d>;
1217 // FIXME: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on CI.
1218 //defm BUFFER_ATOMIC_FCMPSWAP_X2  : MUBUF_Real_Atomic_si <0x5e">;   // isn't on VI
1219 //defm BUFFER_ATOMIC_FMIN_X2      : MUBUF_Real_Atomic_si <0x5f>;    // isn't on VI
1220 //defm BUFFER_ATOMIC_FMAX_X2      : MUBUF_Real_Atomic_si <0x60>;    // isn't on VI
1221
1222 def BUFFER_WBINVL1_SC_si        : MUBUF_Real_si <0x70, BUFFER_WBINVL1_SC>;
1223 def BUFFER_WBINVL1_si           : MUBUF_Real_si <0x71, BUFFER_WBINVL1>;
1224
1225 class MTBUF_Real_si <bits<3> op, MTBUF_Pseudo ps> :
1226   MTBUF_Real<ps>,
1227   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> {
1228   let AssemblerPredicate=isSICI;
1229   let DecoderNamespace="SICI";
1230
1231   bits<1> addr64;
1232   let Inst{15}    = addr64;
1233   let Inst{18-16} = op;
1234 }
1235
1236 def TBUFFER_LOAD_FORMAT_XYZW_si  : MTBUF_Real_si <3, TBUFFER_LOAD_FORMAT_XYZW>;
1237 def TBUFFER_STORE_FORMAT_X_si    : MTBUF_Real_si <4, TBUFFER_STORE_FORMAT_X>;
1238 def TBUFFER_STORE_FORMAT_XY_si   : MTBUF_Real_si <5, TBUFFER_STORE_FORMAT_XY>;
1239 def TBUFFER_STORE_FORMAT_XYZ_si  : MTBUF_Real_si <6, TBUFFER_STORE_FORMAT_XYZ>;
1240 def TBUFFER_STORE_FORMAT_XYZW_si : MTBUF_Real_si <7, TBUFFER_STORE_FORMAT_XYZW>;
1241
1242
1243 //===----------------------------------------------------------------------===//
1244 // CI
1245 //===----------------------------------------------------------------------===//
1246
1247 class MUBUF_Real_ci <bits<7> op, MUBUF_Pseudo ps> :
1248   MUBUF_Real_si<op, ps> {
1249   let AssemblerPredicate=isCIOnly;
1250   let DecoderNamespace="CI";
1251 }
1252
1253 def BUFFER_WBINVL1_VOL_ci : MUBUF_Real_ci <0x70, BUFFER_WBINVL1_VOL>;
1254
1255
1256 //===----------------------------------------------------------------------===//
1257 // VI
1258 //===----------------------------------------------------------------------===//
1259
1260 class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> :
1261   MUBUF_Real<op, ps>,
1262   Enc64,
1263   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
1264   let AssemblerPredicate=isVI;
1265   let DecoderNamespace="VI";
1266
1267   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1268   let Inst{12}    = ps.offen;
1269   let Inst{13}    = ps.idxen;
1270   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1271   let Inst{16}    = lds;
1272   let Inst{17}    = !if(ps.has_slc, slc, ?);
1273   let Inst{24-18} = op;
1274   let Inst{31-26} = 0x38; //encoding
1275   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1276   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1277   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1278   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1279   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1280 }
1281
1282 multiclass MUBUF_Real_AllAddr_vi<bits<7> op> {
1283   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1284   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1285   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1286   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1287 }
1288
1289 multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
1290   MUBUF_Real_AllAddr_vi<op> {
1291   def _RTN_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFSET")>;
1292   def _RTN_OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFEN")>;
1293   def _RTN_IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_IDXEN")>;
1294   def _RTN_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_BOTHEN")>;
1295 }
1296
1297 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_vi <0x00>;
1298 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_vi <0x01>;
1299 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_vi <0x02>;
1300 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_vi <0x03>;
1301 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_vi <0x04>;
1302 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_vi <0x05>;
1303 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_vi <0x06>;
1304 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_vi <0x07>;
1305 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_vi <0x10>;
1306 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_vi <0x11>;
1307 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_vi <0x12>;
1308 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_vi <0x13>;
1309 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_vi <0x14>;
1310 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_vi <0x15>;
1311 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_vi <0x16>;
1312 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_vi <0x17>;
1313 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_vi <0x18>;
1314 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_vi <0x1a>;
1315 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_vi <0x1c>;
1316 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_vi <0x1d>;
1317 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_vi <0x1e>;
1318 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_vi <0x1f>;
1319
1320 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_vi <0x40>;
1321 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_vi <0x41>;
1322 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_vi <0x42>;
1323 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_vi <0x43>;
1324 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_vi <0x44>;
1325 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_vi <0x45>;
1326 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_vi <0x46>;
1327 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_vi <0x47>;
1328 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_vi <0x48>;
1329 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_vi <0x49>;
1330 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_vi <0x4a>;
1331 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_vi <0x4b>;
1332 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_vi <0x4c>;
1333
1334 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_vi <0x60>;
1335 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_vi <0x61>;
1336 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_vi <0x62>;
1337 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_vi <0x63>;
1338 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_vi <0x64>;
1339 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_vi <0x65>;
1340 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_vi <0x66>;
1341 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_vi <0x67>;
1342 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_vi <0x68>;
1343 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_vi <0x69>;
1344 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_vi <0x6a>;
1345 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_vi <0x6b>;
1346 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_vi <0x6c>;
1347
1348 def BUFFER_WBINVL1_vi           : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>;
1349 def BUFFER_WBINVL1_VOL_vi       : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>;
1350
1351 class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
1352   MTBUF_Real<ps>,
1353   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
1354   let AssemblerPredicate=isVI;
1355   let DecoderNamespace="VI";
1356
1357   let Inst{18-15} = op;
1358 }
1359
1360 def TBUFFER_LOAD_FORMAT_XYZW_vi  : MTBUF_Real_vi <3, TBUFFER_LOAD_FORMAT_XYZW>;
1361 def TBUFFER_STORE_FORMAT_X_vi    : MTBUF_Real_vi <4, TBUFFER_STORE_FORMAT_X>;
1362 def TBUFFER_STORE_FORMAT_XY_vi   : MTBUF_Real_vi <5, TBUFFER_STORE_FORMAT_XY>;
1363 def TBUFFER_STORE_FORMAT_XYZ_vi  : MTBUF_Real_vi <6, TBUFFER_STORE_FORMAT_XYZ>;
1364 def TBUFFER_STORE_FORMAT_XYZW_vi : MTBUF_Real_vi <7, TBUFFER_STORE_FORMAT_XYZW>;
1365