contrib/llvm/lib/Target/AMDGPU/BUFInstructions.td

   1 //===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9
  10 def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">;
  11 def MUBUFAddr64 : ComplexPattern<i64, 7, "SelectMUBUFAddr64">;
  12 def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">;
  13
  14 def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantRoot]>;
  15 def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantRoot], 20>;
  16
  17 def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">;
  18 def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
  19 def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">;
  20 def MUBUFIntrinsicOffset : ComplexPattern<i32, 2, "SelectMUBUFIntrinsicOffset">;
  21 def MUBUFIntrinsicVOffset : ComplexPattern<i32, 3, "SelectMUBUFIntrinsicVOffset">;
  22
  23 class MubufLoad <SDPatternOperator op> : PatFrag <
  24   (ops node:$ptr), (op node:$ptr), [{
  25   auto const AS = cast<MemSDNode>(N)->getAddressSpace();
  26   return AS == AMDGPUASI.GLOBAL_ADDRESS ||
  27          AS == AMDGPUASI.CONSTANT_ADDRESS;
  28 }]>;
  29
  30 def mubuf_load          : MubufLoad <load>;
  31 def mubuf_az_extloadi8  : MubufLoad <az_extloadi8>;
  32 def mubuf_sextloadi8    : MubufLoad <sextloadi8>;
  33 def mubuf_az_extloadi16 : MubufLoad <az_extloadi16>;
  34 def mubuf_sextloadi16   : MubufLoad <sextloadi16>;
  35 def mubuf_load_atomic   : MubufLoad <atomic_load>;
  36
  37 def BUFAddrKind {
  38   int Offset = 0;
  39   int OffEn  = 1;
  40   int IdxEn  = 2;
  41   int BothEn = 3;
  42   int Addr64 = 4;
  43 }
  44
  45 class getAddrName<int addrKind> {
  46   string ret =
  47     !if(!eq(addrKind, BUFAddrKind.Offset), "offset",
  48     !if(!eq(addrKind, BUFAddrKind.OffEn),  "offen",
  49     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "idxen",
  50     !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",
  51     !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",
  52     "")))));
  53 }
  54
  55 class MUBUFAddr64Table <bit is_addr64, string suffix = ""> {
  56   bit IsAddr64 = is_addr64;
  57   string OpName = NAME # suffix;
  58 }
  59
  60 class MTBUFAddr64Table <bit is_addr64, string suffix = ""> {
  61   bit IsAddr64 = is_addr64;
  62   string OpName = NAME # suffix;
  63 }
  64
  65 //===----------------------------------------------------------------------===//
  66 // MTBUF classes
  67 //===----------------------------------------------------------------------===//
  68
  69 class MTBUF_Pseudo <string opName, dag outs, dag ins,
  70                     string asmOps, list<dag> pattern=[]> :
  71   InstSI<outs, ins, "", pattern>,
  72   SIMCInstr<opName, SIEncodingFamily.NONE> {
  73
  74   let isPseudo = 1;
  75   let isCodeGenOnly = 1;
  76   let Size = 8;
  77   let UseNamedOperandTable = 1;
  78
  79   string Mnemonic = opName;
  80   string AsmOperands = asmOps;
  81
  82   let VM_CNT = 1;
  83   let EXP_CNT = 1;
  84   let MTBUF = 1;
  85   let Uses = [EXEC];
  86   let hasSideEffects = 0;
  87   let SchedRW = [WriteVMEM];
  88
  89   let AsmMatchConverter = "cvtMtbuf";
  90
  91   bits<1> offen       = 0;
  92   bits<1> idxen       = 0;
  93   bits<1> addr64      = 0;
  94   bits<1> has_vdata   = 1;
  95   bits<1> has_vaddr   = 1;
  96   bits<1> has_glc     = 1;
  97   bits<1> glc_value   = 0; // the value for glc if no such operand
  98   bits<4> dfmt_value  = 1; // the value for dfmt if no such operand
  99   bits<3> nfmt_value  = 0; // the value for nfmt if no such operand
 100   bits<1> has_srsrc   = 1;
 101   bits<1> has_soffset = 1;
 102   bits<1> has_offset  = 1;
 103   bits<1> has_slc     = 1;
 104   bits<1> has_tfe     = 1;
 105   bits<1> has_dfmt    = 1;
 106   bits<1> has_nfmt    = 1;
 107 }
 108
 109 class MTBUF_Real <MTBUF_Pseudo ps> :
 110   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 111
 112   let isPseudo = 0;
 113   let isCodeGenOnly = 0;
 114
 115   // copy relevant pseudo op flags
 116   let SubtargetPredicate = ps.SubtargetPredicate;
 117   let AsmMatchConverter  = ps.AsmMatchConverter;
 118   let Constraints        = ps.Constraints;
 119   let DisableEncoding    = ps.DisableEncoding;
 120   let TSFlags            = ps.TSFlags;
 121
 122   bits<12> offset;
 123   bits<1>  glc;
 124   bits<4>  dfmt;
 125   bits<3>  nfmt;
 126   bits<8>  vaddr;
 127   bits<8>  vdata;
 128   bits<7>  srsrc;
 129   bits<1>  slc;
 130   bits<1>  tfe;
 131   bits<8>  soffset;
 132 }
 133
 134 class getMTBUFInsDA<list<RegisterClass> vdataList,
 135                     list<RegisterClass> vaddrList=[]> {
 136   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 137   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 138   dag InsNoData = !if(!empty(vaddrList),
 139     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 140          offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, slc:$slc, tfe:$tfe),
 141     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 142          offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, slc:$slc, tfe:$tfe)
 143   );
 144   dag InsData = !if(!empty(vaddrList),
 145     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 146          SCSrc_b32:$soffset, offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc,
 147          slc:$slc, tfe:$tfe),
 148     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 149          SCSrc_b32:$soffset, offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc,
 150          slc:$slc, tfe:$tfe)
 151   );
 152   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 153 }
 154
 155 class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
 156   dag ret =
 157     !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret,
 158     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 159     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 160     !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 161     !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 162     (ins))))));
 163 }
 164
 165 class getMTBUFAsmOps<int addrKind> {
 166   string Pfx =
 167     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $dfmt, $nfmt, $soffset",
 168     !if(!eq(addrKind, BUFAddrKind.OffEn),
 169             "$vaddr, $srsrc, $dfmt, $nfmt, $soffset offen",
 170     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 171             "$vaddr, $srsrc, $dfmt, $nfmt, $soffset idxen",
 172     !if(!eq(addrKind, BUFAddrKind.BothEn),
 173             "$vaddr, $srsrc, $dfmt, $nfmt, $soffset idxen offen",
 174     !if(!eq(addrKind, BUFAddrKind.Addr64),
 175             "$vaddr, $srsrc, $dfmt, $nfmt, $soffset addr64",
 176     "")))));
 177   string ret = Pfx # "$offset";
 178 }
 179
 180 class MTBUF_SetupAddr<int addrKind> {
 181   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 182                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 183
 184   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 185                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 186
 187   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 188
 189   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 190 }
 191
 192 class MTBUF_Load_Pseudo <string opName,
 193                          int addrKind,
 194                          RegisterClass vdataClass,
 195                          list<dag> pattern=[],
 196                          // Workaround bug bz30254
 197                          int addrKindCopy = addrKind>
 198   : MTBUF_Pseudo<opName,
 199                  (outs vdataClass:$vdata),
 200                  getMTBUFIns<addrKindCopy>.ret,
 201                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
 202                  pattern>,
 203     MTBUF_SetupAddr<addrKindCopy> {
 204   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 205   let mayLoad = 1;
 206   let mayStore = 0;
 207 }
 208
 209 multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 210                               ValueType load_vt = i32,
 211                               SDPatternOperator ld = null_frag> {
 212
 213   def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 214     [(set load_vt:$vdata,
 215      (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$dfmt,
 216                       i8:$nfmt, i1:$glc, i1:$slc, i1:$tfe)))]>,
 217     MTBUFAddr64Table<0>;
 218
 219   def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 220     [(set load_vt:$vdata,
 221      (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset,
 222                       i8:$dfmt, i8:$nfmt, i1:$glc, i1:$slc, i1:$tfe)))]>,
 223     MTBUFAddr64Table<1>;
 224
 225   def _OFFEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 226   def _IDXEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 227   def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 228
 229   let DisableWQM = 1 in {
 230     def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 231     def _OFFEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 232     def _IDXEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 233     def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 234   }
 235 }
 236
 237 class MTBUF_Store_Pseudo <string opName,
 238                           int addrKind,
 239                           RegisterClass vdataClass,
 240                           list<dag> pattern=[],
 241                           // Workaround bug bz30254
 242                           int addrKindCopy = addrKind,
 243                           RegisterClass vdataClassCopy = vdataClass>
 244   : MTBUF_Pseudo<opName,
 245                  (outs),
 246                  getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 247                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
 248                  pattern>,
 249     MTBUF_SetupAddr<addrKindCopy> {
 250   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 251   let mayLoad = 0;
 252   let mayStore = 1;
 253 }
 254
 255 multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 256                                ValueType store_vt = i32,
 257                                SDPatternOperator st = null_frag> {
 258
 259   def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 260     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 261                                        i16:$offset, i8:$dfmt, i8:$nfmt, i1:$glc,
 262                                        i1:$slc, i1:$tfe))]>,
 263     MTBUFAddr64Table<0>;
 264
 265   def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 266     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 267                                        i16:$offset, i8:$dfmt, i8:$nfmt, i1:$glc,
 268                                        i1:$slc, i1:$tfe))]>,
 269     MTBUFAddr64Table<1>;
 270
 271   def _OFFEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 272   def _IDXEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 273   def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 274
 275   let DisableWQM = 1 in {
 276     def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 277     def _OFFEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 278     def _IDXEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 279     def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 280   }
 281 }
 282
 283
 284 //===----------------------------------------------------------------------===//
 285 // MUBUF classes
 286 //===----------------------------------------------------------------------===//
 287
 288 class MUBUF_Pseudo <string opName, dag outs, dag ins,
 289                     string asmOps, list<dag> pattern=[]> :
 290   InstSI<outs, ins, "", pattern>,
 291   SIMCInstr<opName, SIEncodingFamily.NONE> {
 292
 293   let isPseudo = 1;
 294   let isCodeGenOnly = 1;
 295   let Size = 8;
 296   let UseNamedOperandTable = 1;
 297
 298   string Mnemonic = opName;
 299   string AsmOperands = asmOps;
 300
 301   let VM_CNT = 1;
 302   let EXP_CNT = 1;
 303   let MUBUF = 1;
 304   let Uses = [EXEC];
 305   let hasSideEffects = 0;
 306   let SchedRW = [WriteVMEM];
 307
 308   let AsmMatchConverter = "cvtMubuf";
 309
 310   bits<1> offen       = 0;
 311   bits<1> idxen       = 0;
 312   bits<1> addr64      = 0;
 313   bits<1> has_vdata   = 1;
 314   bits<1> has_vaddr   = 1;
 315   bits<1> has_glc     = 1;
 316   bits<1> glc_value   = 0; // the value for glc if no such operand
 317   bits<1> has_srsrc   = 1;
 318   bits<1> has_soffset = 1;
 319   bits<1> has_offset  = 1;
 320   bits<1> has_slc     = 1;
 321   bits<1> has_tfe     = 1;
 322 }
 323
 324 class MUBUF_Real <bits<7> op, MUBUF_Pseudo ps> :
 325   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 326
 327   let isPseudo = 0;
 328   let isCodeGenOnly = 0;
 329
 330   // copy relevant pseudo op flags
 331   let SubtargetPredicate = ps.SubtargetPredicate;
 332   let AsmMatchConverter  = ps.AsmMatchConverter;
 333   let Constraints        = ps.Constraints;
 334   let DisableEncoding    = ps.DisableEncoding;
 335   let TSFlags            = ps.TSFlags;
 336
 337   bits<12> offset;
 338   bits<1>  glc;
 339   bits<1>  lds = 0;
 340   bits<8>  vaddr;
 341   bits<8>  vdata;
 342   bits<7>  srsrc;
 343   bits<1>  slc;
 344   bits<1>  tfe;
 345   bits<8>  soffset;
 346 }
 347
 348
 349 // For cache invalidation instructions.
 350 class MUBUF_Invalidate <string opName, SDPatternOperator node> :
 351   MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {
 352
 353   let AsmMatchConverter = "";
 354
 355   let hasSideEffects = 1;
 356   let mayStore = 1;
 357
 358   // Set everything to 0.
 359   let offen       = 0;
 360   let idxen       = 0;
 361   let addr64      = 0;
 362   let has_vdata   = 0;
 363   let has_vaddr   = 0;
 364   let has_glc     = 0;
 365   let glc_value   = 0;
 366   let has_srsrc   = 0;
 367   let has_soffset = 0;
 368   let has_offset  = 0;
 369   let has_slc     = 0;
 370   let has_tfe     = 0;
 371 }
 372
 373 class getMUBUFInsDA<list<RegisterClass> vdataList,
 374                     list<RegisterClass> vaddrList=[]> {
 375   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 376   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 377   dag InsNoData = !if(!empty(vaddrList),
 378     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 379          offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe),
 380     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 381          offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe)
 382   );
 383   dag InsData = !if(!empty(vaddrList),
 384     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 385          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe),
 386     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 387          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe)
 388   );
 389   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 390 }
 391
 392 class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
 393   dag ret =
 394     !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList>.ret,
 395     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMUBUFInsDA<vdataList, [VGPR_32]>.ret,
 396     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMUBUFInsDA<vdataList, [VGPR_32]>.ret,
 397     !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64]>.ret,
 398     !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64]>.ret,
 399     (ins))))));
 400 }
 401
 402 class getMUBUFAsmOps<int addrKind> {
 403   string Pfx =
 404     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",
 405     !if(!eq(addrKind, BUFAddrKind.OffEn),  "$vaddr, $srsrc, $soffset offen",
 406     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "$vaddr, $srsrc, $soffset idxen",
 407     !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",
 408     !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",
 409     "")))));
 410   string ret = Pfx # "$offset";
 411 }
 412
 413 class MUBUF_SetupAddr<int addrKind> {
 414   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 415                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 416
 417   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 418                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 419
 420   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 421
 422   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 423 }
 424
 425 class MUBUF_Load_Pseudo <string opName,
 426                          int addrKind,
 427                          RegisterClass vdataClass,
 428                          list<dag> pattern=[],
 429                          // Workaround bug bz30254
 430                          int addrKindCopy = addrKind>
 431   : MUBUF_Pseudo<opName,
 432                  (outs vdataClass:$vdata),
 433                  getMUBUFIns<addrKindCopy>.ret,
 434                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
 435                  pattern>,
 436     MUBUF_SetupAddr<addrKindCopy> {
 437   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 438   let mayLoad = 1;
 439   let mayStore = 0;
 440 }
 441
 442 // FIXME: tfe can't be an operand because it requires a separate
 443 // opcode because it needs an N+1 register class dest register.
 444 multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 445                               ValueType load_vt = i32,
 446                               SDPatternOperator ld = null_frag> {
 447
 448   def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 449     [(set load_vt:$vdata,
 450      (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>,
 451     MUBUFAddr64Table<0>;
 452
 453   def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 454     [(set load_vt:$vdata,
 455      (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>,
 456     MUBUFAddr64Table<1>;
 457
 458   def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 459   def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 460   def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 461
 462   let DisableWQM = 1 in {
 463     def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 464     def _OFFEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 465     def _IDXEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 466     def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 467   }
 468 }
 469
 470 class MUBUF_Store_Pseudo <string opName,
 471                           int addrKind,
 472                           RegisterClass vdataClass,
 473                           list<dag> pattern=[],
 474                           // Workaround bug bz30254
 475                           int addrKindCopy = addrKind,
 476                           RegisterClass vdataClassCopy = vdataClass>
 477   : MUBUF_Pseudo<opName,
 478                  (outs),
 479                  getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 480                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
 481                  pattern>,
 482     MUBUF_SetupAddr<addrKindCopy> {
 483   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 484   let mayLoad = 0;
 485   let mayStore = 1;
 486 }
 487
 488 multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 489                                ValueType store_vt = i32,
 490                                SDPatternOperator st = null_frag> {
 491
 492   def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 493     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 494                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>,
 495     MUBUFAddr64Table<0>;
 496
 497   def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 498     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 499                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>,
 500     MUBUFAddr64Table<1>;
 501
 502   def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 503   def _IDXEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 504   def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 505
 506   let DisableWQM = 1 in {
 507     def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 508     def _OFFEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 509     def _IDXEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 510     def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 511   }
 512 }
 513
 514
 515 class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
 516                           list<RegisterClass> vaddrList=[]> {
 517   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 518   dag ret = !if(vdata_in,
 519     !if(!empty(vaddrList),
 520       (ins vdataClass:$vdata_in,
 521            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc),
 522       (ins vdataClass:$vdata_in, vaddrClass:$vaddr,
 523            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc)
 524     ),
 525     !if(!empty(vaddrList),
 526       (ins vdataClass:$vdata,
 527            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc),
 528       (ins vdataClass:$vdata, vaddrClass:$vaddr,
 529            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc)
 530   ));
 531 }
 532
 533 class getMUBUFAtomicIns<int addrKind,
 534                         RegisterClass vdataClass,
 535                         bit vdata_in,
 536                         // Workaround bug bz30254
 537                         RegisterClass vdataClassCopy=vdataClass> {
 538   dag ret =
 539     !if(!eq(addrKind, BUFAddrKind.Offset),
 540             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret,
 541     !if(!eq(addrKind, BUFAddrKind.OffEn),
 542             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 543     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 544             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 545     !if(!eq(addrKind, BUFAddrKind.BothEn),
 546             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 547     !if(!eq(addrKind, BUFAddrKind.Addr64),
 548             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 549     (ins))))));
 550 }
 551
 552 class MUBUF_Atomic_Pseudo<string opName,
 553                           int addrKind,
 554                           dag outs,
 555                           dag ins,
 556                           string asmOps,
 557                           list<dag> pattern=[],
 558                           // Workaround bug bz30254
 559                           int addrKindCopy = addrKind>
 560   : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,
 561     MUBUF_SetupAddr<addrKindCopy> {
 562   let mayStore = 1;
 563   let mayLoad = 1;
 564   let hasPostISelHook = 1;
 565   let hasSideEffects = 1;
 566   let DisableWQM = 1;
 567   let has_glc = 0;
 568   let has_tfe = 0;
 569 }
 570
 571 class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
 572                                RegisterClass vdataClass,
 573                                list<dag> pattern=[],
 574                                // Workaround bug bz30254
 575                                int addrKindCopy = addrKind,
 576                                RegisterClass vdataClassCopy = vdataClass>
 577   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 578                         (outs),
 579                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret,
 580                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc",
 581                         pattern>,
 582     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> {
 583   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 584   let glc_value = 0;
 585   let AsmMatchConverter = "cvtMubufAtomic";
 586 }
 587
 588 class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
 589                              RegisterClass vdataClass,
 590                              list<dag> pattern=[],
 591                              // Workaround bug bz30254
 592                              int addrKindCopy = addrKind,
 593                              RegisterClass vdataClassCopy = vdataClass>
 594   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 595                         (outs vdataClassCopy:$vdata),
 596                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret,
 597                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc",
 598                         pattern>,
 599     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> {
 600   let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret;
 601   let glc_value = 1;
 602   let Constraints = "$vdata = $vdata_in";
 603   let DisableEncoding = "$vdata_in";
 604   let AsmMatchConverter = "cvtMubufAtomicReturn";
 605 }
 606
 607 multiclass MUBUF_Pseudo_Atomics <string opName,
 608                                  RegisterClass vdataClass,
 609                                  ValueType vdataType,
 610                                  SDPatternOperator atomic> {
 611
 612   def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>,
 613                 MUBUFAddr64Table <0>;
 614   def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>,
 615                 MUBUFAddr64Table <1>;
 616   def _OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 617   def _IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 618   def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 619
 620   def _RTN_OFFSET : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 621     [(set vdataType:$vdata,
 622      (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc),
 623              vdataType:$vdata_in))]>,
 624     MUBUFAddr64Table <0, "_RTN">;
 625
 626   def _RTN_ADDR64 : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 627     [(set vdataType:$vdata,
 628      (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc),
 629              vdataType:$vdata_in))]>,
 630     MUBUFAddr64Table <1, "_RTN">;
 631
 632   def _RTN_OFFEN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 633   def _RTN_IDXEN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 634   def _RTN_BOTHEN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 635 }
 636
 637
 638 //===----------------------------------------------------------------------===//
 639 // MUBUF Instructions
 640 //===----------------------------------------------------------------------===//
 641
 642 let SubtargetPredicate = isGCN in {
 643
 644 defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads <
 645   "buffer_load_format_x", VGPR_32
 646 >;
 647 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
 648   "buffer_load_format_xy", VReg_64
 649 >;
 650 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <
 651   "buffer_load_format_xyz", VReg_96
 652 >;
 653 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <
 654   "buffer_load_format_xyzw", VReg_128
 655 >;
 656 defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <
 657   "buffer_store_format_x", VGPR_32
 658 >;
 659 defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <
 660   "buffer_store_format_xy", VReg_64
 661 >;
 662 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <
 663   "buffer_store_format_xyz", VReg_96
 664 >;
 665 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <
 666   "buffer_store_format_xyzw", VReg_128
 667 >;
 668 defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads <
 669   "buffer_load_ubyte", VGPR_32, i32, mubuf_az_extloadi8
 670 >;
 671 defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads <
 672   "buffer_load_sbyte", VGPR_32, i32, mubuf_sextloadi8
 673 >;
 674 defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads <
 675   "buffer_load_ushort", VGPR_32, i32, mubuf_az_extloadi16
 676 >;
 677 defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads <
 678   "buffer_load_sshort", VGPR_32, i32, mubuf_sextloadi16
 679 >;
 680 defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads <
 681   "buffer_load_dword", VGPR_32, i32, mubuf_load
 682 >;
 683 defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
 684   "buffer_load_dwordx2", VReg_64, v2i32, mubuf_load
 685 >;
 686 defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
 687   "buffer_load_dwordx3", VReg_96, untyped, mubuf_load
 688 >;
 689 defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
 690   "buffer_load_dwordx4", VReg_128, v4i32, mubuf_load
 691 >;
 692 defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
 693   "buffer_store_byte", VGPR_32, i32, truncstorei8_global
 694 >;
 695 defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
 696   "buffer_store_short", VGPR_32, i32, truncstorei16_global
 697 >;
 698 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
 699   "buffer_store_dword", VGPR_32, i32, global_store
 700 >;
 701 defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
 702   "buffer_store_dwordx2", VReg_64, v2i32, global_store
 703 >;
 704 defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
 705   "buffer_store_dwordx3", VReg_96, untyped, global_store
 706 >;
 707 defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
 708   "buffer_store_dwordx4", VReg_128, v4i32, global_store
 709 >;
 710 defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
 711   "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global
 712 >;
 713 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <
 714   "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag
 715 >;
 716 defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <
 717   "buffer_atomic_add", VGPR_32, i32, atomic_add_global
 718 >;
 719 defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <
 720   "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global
 721 >;
 722 defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <
 723   "buffer_atomic_smin", VGPR_32, i32, atomic_min_global
 724 >;
 725 defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <
 726   "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global
 727 >;
 728 defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <
 729   "buffer_atomic_smax", VGPR_32, i32, atomic_max_global
 730 >;
 731 defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <
 732   "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global
 733 >;
 734 defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <
 735   "buffer_atomic_and", VGPR_32, i32, atomic_and_global
 736 >;
 737 defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <
 738   "buffer_atomic_or", VGPR_32, i32, atomic_or_global
 739 >;
 740 defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <
 741   "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global
 742 >;
 743 defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <
 744   "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global
 745 >;
 746 defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <
 747   "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global
 748 >;
 749 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <
 750   "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global
 751 >;
 752 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <
 753   "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag
 754 >;
 755 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <
 756   "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global
 757 >;
 758 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <
 759   "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global
 760 >;
 761 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <
 762   "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global
 763 >;
 764 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <
 765   "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global
 766 >;
 767 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <
 768   "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global
 769 >;
 770 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <
 771   "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global
 772 >;
 773 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <
 774   "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global
 775 >;
 776 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <
 777   "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global
 778 >;
 779 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <
 780   "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global
 781 >;
 782 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <
 783   "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global
 784 >;
 785 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
 786   "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global
 787 >;
 788
 789 let SubtargetPredicate = isSI in { // isn't on CI & VI
 790 /*
 791 defm BUFFER_ATOMIC_RSUB        : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
 792 defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">;
 793 defm BUFFER_ATOMIC_FMIN        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">;
 794 defm BUFFER_ATOMIC_FMAX        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">;
 795 defm BUFFER_ATOMIC_RSUB_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
 796 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">;
 797 defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">;
 798 defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">;
 799 */
 800
 801 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
 802                                           int_amdgcn_buffer_wbinvl1_sc>;
 803 }
 804
 805 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
 806                                        int_amdgcn_buffer_wbinvl1>;
 807
 808 //===----------------------------------------------------------------------===//
 809 // MTBUF Instructions
 810 //===----------------------------------------------------------------------===//
 811
 812 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32>;
 813 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xy",    VReg_64>;
 814 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyz",   VReg_128>;
 815 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyzw",  VReg_128>;
 816 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_x",    VGPR_32>;
 817 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy",   VReg_64>;
 818 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz",  VReg_128>;
 819 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>;
 820
 821 } // End let SubtargetPredicate = isGCN
 822
 823 let SubtargetPredicate = isCIVI in {
 824
 825 //===----------------------------------------------------------------------===//
 826 // Instruction definitions for CI and newer.
 827 //===----------------------------------------------------------------------===//
 828 // Remaining instructions:
 829 // BUFFER_LOAD_DWORDX3
 830 // BUFFER_STORE_DWORDX3
 831
 832 def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
 833                                            int_amdgcn_buffer_wbinvl1_vol>;
 834
 835 } // End let SubtargetPredicate = isCIVI
 836
 837 //===----------------------------------------------------------------------===//
 838 // MUBUF Patterns
 839 //===----------------------------------------------------------------------===//
 840
 841 let Predicates = [isGCN] in {
 842
 843 // Offset in an 32-bit VGPR
 844 def : Pat <
 845   (SIload_constant v4i32:$sbase, i32:$voff),
 846   (BUFFER_LOAD_DWORD_OFFEN $voff, $sbase, (i32 0), 0, 0, 0, 0)
 847 >;
 848
 849
 850 //===----------------------------------------------------------------------===//
 851 // buffer_load/store_format patterns
 852 //===----------------------------------------------------------------------===//
 853
 854 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
 855                                   string opcode> {
 856   def : Pat<
 857     (vt (name v4i32:$rsrc, 0,
 858               (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 859               imm:$glc, imm:$slc)),
 860     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
 861       (as_i1imm $glc), (as_i1imm $slc), 0)
 862   >;
 863
 864   def : Pat<
 865     (vt (name v4i32:$rsrc, i32:$vindex,
 866               (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 867               imm:$glc, imm:$slc)),
 868     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
 869       (as_i1imm $glc), (as_i1imm $slc), 0)
 870   >;
 871
 872   def : Pat<
 873     (vt (name v4i32:$rsrc, 0,
 874               (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 875               imm:$glc, imm:$slc)),
 876     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
 877       (as_i1imm $glc), (as_i1imm $slc), 0)
 878   >;
 879
 880   def : Pat<
 881     (vt (name v4i32:$rsrc, i32:$vindex,
 882               (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 883               imm:$glc, imm:$slc)),
 884     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
 885       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
 886       $rsrc, $soffset, (as_i16imm $offset),
 887       (as_i1imm $glc), (as_i1imm $slc), 0)
 888   >;
 889 }
 890
 891 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
 892 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
 893 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;
 894 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">;
 895 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">;
 896 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">;
 897
 898 multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
 899                                    string opcode> {
 900   def : Pat<
 901     (name vt:$vdata, v4i32:$rsrc, 0,
 902           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 903           imm:$glc, imm:$slc),
 904     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset),
 905                                     (as_i1imm $glc), (as_i1imm $slc), 0)
 906   >;
 907
 908   def : Pat<
 909     (name vt:$vdata, v4i32:$rsrc, i32:$vindex,
 910           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 911           imm:$glc, imm:$slc),
 912     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
 913                                    (as_i16imm $offset), (as_i1imm $glc),
 914                                    (as_i1imm $slc), 0)
 915   >;
 916
 917   def : Pat<
 918     (name vt:$vdata, v4i32:$rsrc, 0,
 919           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 920           imm:$glc, imm:$slc),
 921     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
 922                                    (as_i16imm $offset), (as_i1imm $glc),
 923                                    (as_i1imm $slc), 0)
 924   >;
 925
 926   def : Pat<
 927     (name vt:$vdata, v4i32:$rsrc, i32:$vindex,
 928           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 929           imm:$glc, imm:$slc),
 930     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
 931       $vdata,
 932       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
 933       $rsrc, $soffset, (as_i16imm $offset),
 934       (as_i1imm $glc), (as_i1imm $slc), 0)
 935   >;
 936 }
 937
 938 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
 939 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
 940 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
 941 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, f32, "BUFFER_STORE_DWORD">;
 942 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v2f32, "BUFFER_STORE_DWORDX2">;
 943 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v4f32, "BUFFER_STORE_DWORDX4">;
 944
 945 //===----------------------------------------------------------------------===//
 946 // buffer_atomic patterns
 947 //===----------------------------------------------------------------------===//
 948
 949 multiclass BufferAtomicPatterns<SDPatternOperator name, string opcode> {
 950   def : Pat<
 951     (name i32:$vdata_in, v4i32:$rsrc, 0,
 952           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 953           imm:$slc),
 954     (!cast<MUBUF_Pseudo>(opcode # _RTN_OFFSET) $vdata_in, $rsrc, $soffset,
 955                                         (as_i16imm $offset), (as_i1imm $slc))
 956   >;
 957
 958   def : Pat<
 959     (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex,
 960           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 961           imm:$slc),
 962     (!cast<MUBUF_Pseudo>(opcode # _RTN_IDXEN) $vdata_in, $vindex, $rsrc, $soffset,
 963                                        (as_i16imm $offset), (as_i1imm $slc))
 964   >;
 965
 966   def : Pat<
 967     (name i32:$vdata_in, v4i32:$rsrc, 0,
 968           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 969           imm:$slc),
 970     (!cast<MUBUF_Pseudo>(opcode # _RTN_OFFEN) $vdata_in, $voffset, $rsrc, $soffset,
 971                                        (as_i16imm $offset), (as_i1imm $slc))
 972   >;
 973
 974   def : Pat<
 975     (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex,
 976           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 977           imm:$slc),
 978     (!cast<MUBUF_Pseudo>(opcode # _RTN_BOTHEN)
 979       $vdata_in,
 980       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
 981       $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc))
 982   >;
 983 }
 984
 985 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_swap, "BUFFER_ATOMIC_SWAP">;
 986 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_add, "BUFFER_ATOMIC_ADD">;
 987 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_sub, "BUFFER_ATOMIC_SUB">;
 988 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smin, "BUFFER_ATOMIC_SMIN">;
 989 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umin, "BUFFER_ATOMIC_UMIN">;
 990 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smax, "BUFFER_ATOMIC_SMAX">;
 991 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umax, "BUFFER_ATOMIC_UMAX">;
 992 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_and, "BUFFER_ATOMIC_AND">;
 993 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_or, "BUFFER_ATOMIC_OR">;
 994 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_xor, "BUFFER_ATOMIC_XOR">;
 995
 996 def : Pat<
 997   (int_amdgcn_buffer_atomic_cmpswap
 998       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
 999       (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
1000       imm:$slc),
1001   (EXTRACT_SUBREG
1002     (BUFFER_ATOMIC_CMPSWAP_RTN_OFFSET
1003       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1004       $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
1005     sub0)
1006 >;
1007
1008 def : Pat<
1009   (int_amdgcn_buffer_atomic_cmpswap
1010       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1011       (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
1012       imm:$slc),
1013   (EXTRACT_SUBREG
1014     (BUFFER_ATOMIC_CMPSWAP_RTN_IDXEN
1015       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1016       $vindex, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
1017     sub0)
1018 >;
1019
1020 def : Pat<
1021   (int_amdgcn_buffer_atomic_cmpswap
1022       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1023       (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
1024       imm:$slc),
1025   (EXTRACT_SUBREG
1026     (BUFFER_ATOMIC_CMPSWAP_RTN_OFFEN
1027       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1028       $voffset, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
1029     sub0)
1030 >;
1031
1032 def : Pat<
1033   (int_amdgcn_buffer_atomic_cmpswap
1034       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1035       (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
1036       imm:$slc),
1037   (EXTRACT_SUBREG
1038     (BUFFER_ATOMIC_CMPSWAP_RTN_BOTHEN
1039       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1040       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1041       $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
1042     sub0)
1043 >;
1044
1045
1046 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
1047                               PatFrag constant_ld> : Pat <
1048      (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1049                                    i16:$offset, i1:$glc, i1:$slc, i1:$tfe))),
1050      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe)
1051   >;
1052
1053 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1054                                      ValueType vt, PatFrag atomic_ld> {
1055   def : Pat <
1056      (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1057                                    i16:$offset, i1:$slc))),
1058      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 1, $slc, 0)
1059   >;
1060
1061   def : Pat <
1062     (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))),
1063     (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 1, 0, 0)
1064   >;
1065 }
1066
1067 let Predicates = [isSICI] in {
1068 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
1069 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, az_extloadi8_constant>;
1070 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
1071 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, az_extloadi16_constant>;
1072
1073 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, mubuf_load_atomic>;
1074 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, mubuf_load_atomic>;
1075 } // End Predicates = [isSICI]
1076
1077 multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1078                                PatFrag ld> {
1079
1080   def : Pat <
1081     (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1082                           i16:$offset, i1:$glc, i1:$slc, i1:$tfe))),
1083     (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe)
1084   >;
1085 }
1086
1087 let Predicates = [Has16BitInsts] in {
1088
1089 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>;
1090 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_constant>;
1091 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, mubuf_sextloadi8>;
1092 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, mubuf_az_extloadi8>;
1093
1094 } // End Predicates = [Has16BitInsts]
1095
1096 multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen,
1097                                 MUBUF_Pseudo InstrOffset,
1098                                 ValueType vt, PatFrag ld> {
1099   def : Pat <
1100     (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1101                                i32:$soffset, u16imm:$offset))),
1102     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
1103   >;
1104
1105   def : Pat <
1106     (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))),
1107     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0)
1108   >;
1109 }
1110
1111 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>;
1112 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>;
1113 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>;
1114 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>;
1115 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>;
1116 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>;
1117 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>;
1118 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>;
1119 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>;
1120
1121 // BUFFER_LOAD_DWORD*, addr64=0
1122 multiclass MUBUF_Load_Dword <ValueType vt,
1123                              MUBUF_Pseudo offset,
1124                              MUBUF_Pseudo offen,
1125                              MUBUF_Pseudo idxen,
1126                              MUBUF_Pseudo bothen> {
1127
1128   def : Pat <
1129     (vt (int_SI_buffer_load_dword v4i32:$rsrc, (i32 imm), i32:$soffset,
1130                                   imm:$offset, 0, 0, imm:$glc, imm:$slc,
1131                                   imm:$tfe)),
1132     (offset $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc),
1133             (as_i1imm $slc), (as_i1imm $tfe))
1134   >;
1135
1136   def : Pat <
1137     (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset,
1138                                   imm:$offset, 1, 0, imm:$glc, imm:$slc,
1139                                   imm:$tfe)),
1140     (offen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc),
1141            (as_i1imm $tfe))
1142   >;
1143
1144   def : Pat <
1145     (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset,
1146                                   imm:$offset, 0, 1, imm:$glc, imm:$slc,
1147                                   imm:$tfe)),
1148     (idxen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc),
1149            (as_i1imm $slc), (as_i1imm $tfe))
1150   >;
1151
1152   def : Pat <
1153     (vt (int_SI_buffer_load_dword v4i32:$rsrc, v2i32:$vaddr, i32:$soffset,
1154                                   imm:$offset, 1, 1, imm:$glc, imm:$slc,
1155                                   imm:$tfe)),
1156     (bothen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc),
1157             (as_i1imm $tfe))
1158   >;
1159 }
1160
1161 defm : MUBUF_Load_Dword <i32, BUFFER_LOAD_DWORD_OFFSET, BUFFER_LOAD_DWORD_OFFEN,
1162                          BUFFER_LOAD_DWORD_IDXEN, BUFFER_LOAD_DWORD_BOTHEN>;
1163 defm : MUBUF_Load_Dword <v2i32, BUFFER_LOAD_DWORDX2_OFFSET, BUFFER_LOAD_DWORDX2_OFFEN,
1164                          BUFFER_LOAD_DWORDX2_IDXEN, BUFFER_LOAD_DWORDX2_BOTHEN>;
1165 defm : MUBUF_Load_Dword <v4i32, BUFFER_LOAD_DWORDX4_OFFSET, BUFFER_LOAD_DWORDX4_OFFEN,
1166                          BUFFER_LOAD_DWORDX4_IDXEN, BUFFER_LOAD_DWORDX4_BOTHEN>;
1167
1168 multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1169                                       ValueType vt, PatFrag atomic_st> {
1170   // Store follows atomic op convention so address is forst
1171   def : Pat <
1172      (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1173                                    i16:$offset, i1:$slc), vt:$val),
1174      (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 1, $slc, 0)
1175   >;
1176
1177   def : Pat <
1178     (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
1179     (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 1, 0, 0)
1180   >;
1181 }
1182 let Predicates = [isSICI] in {
1183 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, global_store_atomic>;
1184 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, global_store_atomic>;
1185 } // End Predicates = [isSICI]
1186
1187
1188 multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1189                                PatFrag st> {
1190
1191   def : Pat <
1192     (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1193                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe)),
1194     (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe)
1195   >;
1196 }
1197
1198 defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>;
1199 defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, global_store>;
1200
1201 multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen,
1202                                  MUBUF_Pseudo InstrOffset,
1203                                  ValueType vt, PatFrag st> {
1204   def : Pat <
1205     (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1206                                       i32:$soffset, u16imm:$offset)),
1207     (InstrOffen $value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
1208   >;
1209
1210   def : Pat <
1211     (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,
1212                                        u16imm:$offset)),
1213     (InstrOffset $value, $srsrc, $soffset, $offset, 0, 0, 0)
1214   >;
1215 }
1216
1217 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>;
1218 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>;
1219 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>;
1220 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>;
1221 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>;
1222 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private>;
1223 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private>;
1224
1225 //===----------------------------------------------------------------------===//
1226 // MTBUF Patterns
1227 //===----------------------------------------------------------------------===//
1228
1229 //===----------------------------------------------------------------------===//
1230 // tbuffer_load/store_format patterns
1231 //===----------------------------------------------------------------------===//
1232
1233 multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1234                                   string opcode> {
1235   def : Pat<
1236     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1237               imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)),
1238     (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
1239       (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1240   >;
1241
1242   def : Pat<
1243     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1244               imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)),
1245     (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
1246       (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1247   >;
1248
1249   def : Pat<
1250     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1251               imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)),
1252     (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
1253       (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1254   >;
1255
1256   def : Pat<
1257     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset,
1258               imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)),
1259     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN)
1260       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1261       $rsrc, $soffset, (as_i16imm $offset),
1262       (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1263   >;
1264 }
1265
1266 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32,   "TBUFFER_LOAD_FORMAT_X">;
1267 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;
1268 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">;
1269 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32,   "TBUFFER_LOAD_FORMAT_X">;
1270 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;
1271 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;
1272
1273 multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1274                                    string opcode> {
1275   def : Pat<
1276     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1277           imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc),
1278     (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset,
1279                                 (as_i16imm $offset), (as_i8imm $dfmt),
1280                                 (as_i8imm $nfmt), (as_i1imm $glc),
1281                                 (as_i1imm $slc), 0)
1282   >;
1283
1284   def : Pat<
1285     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1286           imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc),
1287     (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
1288                                    (as_i16imm $offset), (as_i8imm $dfmt),
1289                                    (as_i8imm $nfmt), (as_i1imm $glc),
1290                                    (as_i1imm $slc), 0)
1291   >;
1292
1293   def : Pat<
1294     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1295           imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc),
1296     (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
1297                                    (as_i16imm $offset), (as_i8imm $dfmt),
1298                                    (as_i8imm $nfmt), (as_i1imm $glc),
1299                                    (as_i1imm $slc), 0)
1300   >;
1301
1302   def : Pat<
1303     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset,
1304           imm:$offset, imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc),
1305     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact)
1306       $vdata,
1307       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1308       $rsrc, $soffset, (as_i16imm $offset),
1309       (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1310   >;
1311 }
1312
1313 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32,   "TBUFFER_STORE_FORMAT_X">;
1314 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;
1315 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4i32, "TBUFFER_STORE_FORMAT_XYZ">;
1316 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">;
1317 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32,   "TBUFFER_STORE_FORMAT_X">;
1318 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;
1319 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4f32, "TBUFFER_STORE_FORMAT_XYZ">;
1320 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;
1321
1322 } // End let Predicates = [isGCN]
1323
1324 //===----------------------------------------------------------------------===//
1325 // Target instructions, move to the appropriate target TD file
1326 //===----------------------------------------------------------------------===//
1327
1328 //===----------------------------------------------------------------------===//
1329 // SI
1330 //===----------------------------------------------------------------------===//
1331
1332 class MUBUF_Real_si <bits<7> op, MUBUF_Pseudo ps> :
1333   MUBUF_Real<op, ps>,
1334   Enc64,
1335   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> {
1336   let AssemblerPredicate=isSICI;
1337   let DecoderNamespace="SICI";
1338
1339   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1340   let Inst{12}    = ps.offen;
1341   let Inst{13}    = ps.idxen;
1342   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1343   let Inst{15}    = ps.addr64;
1344   let Inst{16}    = lds;
1345   let Inst{24-18} = op;
1346   let Inst{31-26} = 0x38; //encoding
1347   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1348   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1349   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1350   let Inst{54}    = !if(ps.has_slc, slc, ?);
1351   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1352   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1353 }
1354
1355 multiclass MUBUF_Real_AllAddr_si<bits<7> op> {
1356   def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1357   def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>;
1358   def _OFFEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1359   def _IDXEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1360   def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1361 }
1362
1363 multiclass MUBUF_Real_Atomic_si<bits<7> op> : MUBUF_Real_AllAddr_si<op> {
1364   def _RTN_OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFSET")>;
1365   def _RTN_ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_ADDR64")>;
1366   def _RTN_OFFEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFEN")>;
1367   def _RTN_IDXEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_IDXEN")>;
1368   def _RTN_BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_BOTHEN")>;
1369 }
1370
1371 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_si <0x00>;
1372 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_si <0x01>;
1373 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_si <0x02>;
1374 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_si <0x03>;
1375 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_si <0x04>;
1376 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_si <0x05>;
1377 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_si <0x06>;
1378 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_si <0x07>;
1379 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_si <0x08>;
1380 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_si <0x09>;
1381 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_si <0x0a>;
1382 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_si <0x0b>;
1383 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_si <0x0c>;
1384 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_si <0x0d>;
1385 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_si <0x0e>;
1386 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_si <0x0f>;
1387 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_si <0x18>;
1388 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_si <0x1a>;
1389 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_si <0x1c>;
1390 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_si <0x1d>;
1391 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_si <0x1e>;
1392 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_si <0x1f>;
1393
1394 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_si <0x30>;
1395 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_si <0x31>;
1396 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_si <0x32>;
1397 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_si <0x33>;
1398 //defm BUFFER_ATOMIC_RSUB         : MUBUF_Real_Atomic_si <0x34>;    // isn't on CI & VI
1399 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_si <0x35>;
1400 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_si <0x36>;
1401 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_si <0x37>;
1402 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_si <0x38>;
1403 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_si <0x39>;
1404 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_si <0x3a>;
1405 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_si <0x3b>;
1406 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_si <0x3c>;
1407 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_si <0x3d>;
1408
1409 //defm BUFFER_ATOMIC_FCMPSWAP     : MUBUF_Real_Atomic_si <0x3e>;    // isn't on VI
1410 //defm BUFFER_ATOMIC_FMIN         : MUBUF_Real_Atomic_si <0x3f>;    // isn't on VI
1411 //defm BUFFER_ATOMIC_FMAX         : MUBUF_Real_Atomic_si <0x40>;    // isn't on VI
1412 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_si <0x50>;
1413 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_si <0x51>;
1414 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_si <0x52>;
1415 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_si <0x53>;
1416 //defm BUFFER_ATOMIC_RSUB_X2      : MUBUF_Real_Atomic_si <0x54>;    // isn't on CI & VI
1417 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_si <0x55>;
1418 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_si <0x56>;
1419 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_si <0x57>;
1420 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_si <0x58>;
1421 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_si <0x59>;
1422 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_si <0x5a>;
1423 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_si <0x5b>;
1424 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_si <0x5c>;
1425 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_si <0x5d>;
1426 // FIXME: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on CI.
1427 //defm BUFFER_ATOMIC_FCMPSWAP_X2  : MUBUF_Real_Atomic_si <0x5e">;   // isn't on VI
1428 //defm BUFFER_ATOMIC_FMIN_X2      : MUBUF_Real_Atomic_si <0x5f>;    // isn't on VI
1429 //defm BUFFER_ATOMIC_FMAX_X2      : MUBUF_Real_Atomic_si <0x60>;    // isn't on VI
1430
1431 def BUFFER_WBINVL1_SC_si        : MUBUF_Real_si <0x70, BUFFER_WBINVL1_SC>;
1432 def BUFFER_WBINVL1_si           : MUBUF_Real_si <0x71, BUFFER_WBINVL1>;
1433
1434 class MTBUF_Real_si <bits<3> op, MTBUF_Pseudo ps> :
1435   MTBUF_Real<ps>,
1436   Enc64,
1437   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> {
1438   let AssemblerPredicate=isSICI;
1439   let DecoderNamespace="SICI";
1440
1441   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1442   let Inst{12}    = ps.offen;
1443   let Inst{13}    = ps.idxen;
1444   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1445   let Inst{15}    = ps.addr64;
1446   let Inst{18-16} = op;
1447   let Inst{22-19} = !if(ps.has_dfmt, dfmt, ps.dfmt_value);
1448   let Inst{25-23} = !if(ps.has_nfmt, nfmt, ps.nfmt_value);
1449   let Inst{31-26} = 0x3a; //encoding
1450   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1451   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1452   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1453   let Inst{54}    = !if(ps.has_slc, slc, ?);
1454   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1455   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1456 }
1457
1458 multiclass MTBUF_Real_AllAddr_si<bits<3> op> {
1459   def _OFFSET_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
1460   def _ADDR64_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>;
1461   def _OFFEN_si  : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
1462   def _IDXEN_si  : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
1463   def _BOTHEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
1464 }
1465
1466 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_si <0>;
1467 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_si <1>;
1468 //defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_si <2>;
1469 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_si <3>;
1470 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_si <4>;
1471 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_si <5>;
1472 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_si <6>;
1473 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_si <7>;
1474
1475 //===----------------------------------------------------------------------===//
1476 // CI
1477 //===----------------------------------------------------------------------===//
1478
1479 class MUBUF_Real_ci <bits<7> op, MUBUF_Pseudo ps> :
1480   MUBUF_Real_si<op, ps> {
1481   let AssemblerPredicate=isCIOnly;
1482   let DecoderNamespace="CI";
1483 }
1484
1485 def BUFFER_WBINVL1_VOL_ci : MUBUF_Real_ci <0x70, BUFFER_WBINVL1_VOL>;
1486
1487
1488 //===----------------------------------------------------------------------===//
1489 // VI
1490 //===----------------------------------------------------------------------===//
1491
1492 class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> :
1493   MUBUF_Real<op, ps>,
1494   Enc64,
1495   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
1496   let AssemblerPredicate=isVI;
1497   let DecoderNamespace="VI";
1498
1499   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1500   let Inst{12}    = ps.offen;
1501   let Inst{13}    = ps.idxen;
1502   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1503   let Inst{16}    = lds;
1504   let Inst{17}    = !if(ps.has_slc, slc, ?);
1505   let Inst{24-18} = op;
1506   let Inst{31-26} = 0x38; //encoding
1507   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1508   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1509   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1510   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1511   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1512 }
1513
1514 multiclass MUBUF_Real_AllAddr_vi<bits<7> op> {
1515   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1516   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1517   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1518   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1519 }
1520
1521 multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
1522   MUBUF_Real_AllAddr_vi<op> {
1523   def _RTN_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFSET")>;
1524   def _RTN_OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFEN")>;
1525   def _RTN_IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_IDXEN")>;
1526   def _RTN_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_BOTHEN")>;
1527 }
1528
1529 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_vi <0x00>;
1530 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_vi <0x01>;
1531 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_vi <0x02>;
1532 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_vi <0x03>;
1533 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_vi <0x04>;
1534 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_vi <0x05>;
1535 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_vi <0x06>;
1536 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_vi <0x07>;
1537 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_vi <0x10>;
1538 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_vi <0x11>;
1539 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_vi <0x12>;
1540 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_vi <0x13>;
1541 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_vi <0x14>;
1542 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_vi <0x15>;
1543 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_vi <0x16>;
1544 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_vi <0x17>;
1545 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_vi <0x18>;
1546 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_vi <0x1a>;
1547 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_vi <0x1c>;
1548 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_vi <0x1d>;
1549 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_vi <0x1e>;
1550 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_vi <0x1f>;
1551
1552 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_vi <0x40>;
1553 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_vi <0x41>;
1554 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_vi <0x42>;
1555 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_vi <0x43>;
1556 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_vi <0x44>;
1557 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_vi <0x45>;
1558 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_vi <0x46>;
1559 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_vi <0x47>;
1560 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_vi <0x48>;
1561 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_vi <0x49>;
1562 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_vi <0x4a>;
1563 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_vi <0x4b>;
1564 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_vi <0x4c>;
1565
1566 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_vi <0x60>;
1567 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_vi <0x61>;
1568 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_vi <0x62>;
1569 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_vi <0x63>;
1570 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_vi <0x64>;
1571 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_vi <0x65>;
1572 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_vi <0x66>;
1573 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_vi <0x67>;
1574 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_vi <0x68>;
1575 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_vi <0x69>;
1576 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_vi <0x6a>;
1577 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_vi <0x6b>;
1578 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_vi <0x6c>;
1579
1580 def BUFFER_WBINVL1_vi           : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>;
1581 def BUFFER_WBINVL1_VOL_vi       : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>;
1582
1583 class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
1584   MTBUF_Real<ps>,
1585   Enc64,
1586   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
1587   let AssemblerPredicate=isVI;
1588   let DecoderNamespace="VI";
1589
1590   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1591   let Inst{12}    = ps.offen;
1592   let Inst{13}    = ps.idxen;
1593   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1594   let Inst{18-15} = op;
1595   let Inst{22-19} = !if(ps.has_dfmt, dfmt, ps.dfmt_value);
1596   let Inst{25-23} = !if(ps.has_nfmt, nfmt, ps.nfmt_value);
1597   let Inst{31-26} = 0x3a; //encoding
1598   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1599   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1600   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1601   let Inst{54}    = !if(ps.has_slc, slc, ?);
1602   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1603   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1604 }
1605
1606 multiclass MTBUF_Real_AllAddr_vi<bits<4> op> {
1607   def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
1608   def _OFFEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
1609   def _IDXEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
1610   def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
1611 }
1612
1613 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_vi <0>;
1614 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_vi <1>;
1615 //defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_vi <2>;
1616 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_vi <3>;
1617 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_vi <4>;
1618 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_vi <5>;
1619 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_vi <6>;
1620 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <7>;