contrib/llvm/lib/Target/AMDGPU/BUFInstructions.td

   1 //===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9
  10 def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">;
  11 def MUBUFAddr64 : ComplexPattern<i64, 7, "SelectMUBUFAddr64">;
  12 def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">;
  13
  14 def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>;
  15 def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>;
  16
  17 def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">;
  18 def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
  19 def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">;
  20 def MUBUFIntrinsicOffset : ComplexPattern<i32, 2, "SelectMUBUFIntrinsicOffset">;
  21 def MUBUFIntrinsicVOffset : ComplexPattern<i32, 3, "SelectMUBUFIntrinsicVOffset">;
  22
  23 class MubufLoad <SDPatternOperator op> : PatFrag <
  24   (ops node:$ptr), (op node:$ptr), [{
  25   auto const AS = cast<MemSDNode>(N)->getAddressSpace();
  26   return AS == AMDGPUASI.GLOBAL_ADDRESS ||
  27          AS == AMDGPUASI.CONSTANT_ADDRESS;
  28 }]>;
  29
  30 def mubuf_load          : MubufLoad <load>;
  31 def mubuf_az_extloadi8  : MubufLoad <az_extloadi8>;
  32 def mubuf_sextloadi8    : MubufLoad <sextloadi8>;
  33 def mubuf_az_extloadi16 : MubufLoad <az_extloadi16>;
  34 def mubuf_sextloadi16   : MubufLoad <sextloadi16>;
  35 def mubuf_load_atomic   : MubufLoad <atomic_load>;
  36
  37 def BUFAddrKind {
  38   int Offset = 0;
  39   int OffEn  = 1;
  40   int IdxEn  = 2;
  41   int BothEn = 3;
  42   int Addr64 = 4;
  43 }
  44
  45 class getAddrName<int addrKind> {
  46   string ret =
  47     !if(!eq(addrKind, BUFAddrKind.Offset), "offset",
  48     !if(!eq(addrKind, BUFAddrKind.OffEn),  "offen",
  49     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "idxen",
  50     !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",
  51     !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",
  52     "")))));
  53 }
  54
  55 class MUBUFAddr64Table <bit is_addr64, string suffix = ""> {
  56   bit IsAddr64 = is_addr64;
  57   string OpName = NAME # suffix;
  58 }
  59
  60 class MTBUFAddr64Table <bit is_addr64, string suffix = ""> {
  61   bit IsAddr64 = is_addr64;
  62   string OpName = NAME # suffix;
  63 }
  64
  65 //===----------------------------------------------------------------------===//
  66 // MTBUF classes
  67 //===----------------------------------------------------------------------===//
  68
  69 class MTBUF_Pseudo <string opName, dag outs, dag ins,
  70                     string asmOps, list<dag> pattern=[]> :
  71   InstSI<outs, ins, "", pattern>,
  72   SIMCInstr<opName, SIEncodingFamily.NONE> {
  73
  74   let isPseudo = 1;
  75   let isCodeGenOnly = 1;
  76   let Size = 8;
  77   let UseNamedOperandTable = 1;
  78
  79   string Mnemonic = opName;
  80   string AsmOperands = asmOps;
  81
  82   let VM_CNT = 1;
  83   let EXP_CNT = 1;
  84   let MTBUF = 1;
  85   let Uses = [EXEC];
  86   let hasSideEffects = 0;
  87   let SchedRW = [WriteVMEM];
  88
  89   let AsmMatchConverter = "cvtMtbuf";
  90
  91   bits<1> offen       = 0;
  92   bits<1> idxen       = 0;
  93   bits<1> addr64      = 0;
  94   bits<1> has_vdata   = 1;
  95   bits<1> has_vaddr   = 1;
  96   bits<1> has_glc     = 1;
  97   bits<1> glc_value   = 0; // the value for glc if no such operand
  98   bits<4> dfmt_value  = 1; // the value for dfmt if no such operand
  99   bits<3> nfmt_value  = 0; // the value for nfmt if no such operand
 100   bits<1> has_srsrc   = 1;
 101   bits<1> has_soffset = 1;
 102   bits<1> has_offset  = 1;
 103   bits<1> has_slc     = 1;
 104   bits<1> has_tfe     = 1;
 105   bits<1> has_dfmt    = 1;
 106   bits<1> has_nfmt    = 1;
 107 }
 108
 109 class MTBUF_Real <MTBUF_Pseudo ps> :
 110   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 111
 112   let isPseudo = 0;
 113   let isCodeGenOnly = 0;
 114
 115   // copy relevant pseudo op flags
 116   let SubtargetPredicate = ps.SubtargetPredicate;
 117   let AsmMatchConverter  = ps.AsmMatchConverter;
 118   let Constraints        = ps.Constraints;
 119   let DisableEncoding    = ps.DisableEncoding;
 120   let TSFlags            = ps.TSFlags;
 121
 122   bits<12> offset;
 123   bits<1>  glc;
 124   bits<4>  dfmt;
 125   bits<3>  nfmt;
 126   bits<8>  vaddr;
 127   bits<8>  vdata;
 128   bits<7>  srsrc;
 129   bits<1>  slc;
 130   bits<1>  tfe;
 131   bits<8>  soffset;
 132 }
 133
 134 class getMTBUFInsDA<list<RegisterClass> vdataList,
 135                     list<RegisterClass> vaddrList=[]> {
 136   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 137   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 138   dag InsNoData = !if(!empty(vaddrList),
 139     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 140          offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, slc:$slc, tfe:$tfe),
 141     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 142          offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, slc:$slc, tfe:$tfe)
 143   );
 144   dag InsData = !if(!empty(vaddrList),
 145     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 146          SCSrc_b32:$soffset, offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc,
 147          slc:$slc, tfe:$tfe),
 148     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 149          SCSrc_b32:$soffset, offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc,
 150          slc:$slc, tfe:$tfe)
 151   );
 152   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 153 }
 154
 155 class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
 156   dag ret =
 157     !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret,
 158     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 159     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 160     !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 161     !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 162     (ins))))));
 163 }
 164
 165 class getMTBUFAsmOps<int addrKind> {
 166   string Pfx =
 167     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $dfmt, $nfmt, $soffset",
 168     !if(!eq(addrKind, BUFAddrKind.OffEn),
 169             "$vaddr, $srsrc, $dfmt, $nfmt, $soffset offen",
 170     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 171             "$vaddr, $srsrc, $dfmt, $nfmt, $soffset idxen",
 172     !if(!eq(addrKind, BUFAddrKind.BothEn),
 173             "$vaddr, $srsrc, $dfmt, $nfmt, $soffset idxen offen",
 174     !if(!eq(addrKind, BUFAddrKind.Addr64),
 175             "$vaddr, $srsrc, $dfmt, $nfmt, $soffset addr64",
 176     "")))));
 177   string ret = Pfx # "$offset";
 178 }
 179
 180 class MTBUF_SetupAddr<int addrKind> {
 181   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 182                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 183
 184   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 185                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 186
 187   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 188
 189   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 190 }
 191
 192 class MTBUF_Load_Pseudo <string opName,
 193                          int addrKind,
 194                          RegisterClass vdataClass,
 195                          list<dag> pattern=[],
 196                          // Workaround bug bz30254
 197                          int addrKindCopy = addrKind>
 198   : MTBUF_Pseudo<opName,
 199                  (outs vdataClass:$vdata),
 200                  getMTBUFIns<addrKindCopy>.ret,
 201                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
 202                  pattern>,
 203     MTBUF_SetupAddr<addrKindCopy> {
 204   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 205   let mayLoad = 1;
 206   let mayStore = 0;
 207 }
 208
 209 multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 210                               ValueType load_vt = i32,
 211                               SDPatternOperator ld = null_frag> {
 212
 213   def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 214     [(set load_vt:$vdata,
 215      (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$dfmt,
 216                       i8:$nfmt, i1:$glc, i1:$slc, i1:$tfe)))]>,
 217     MTBUFAddr64Table<0>;
 218
 219   def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 220     [(set load_vt:$vdata,
 221      (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset,
 222                       i8:$dfmt, i8:$nfmt, i1:$glc, i1:$slc, i1:$tfe)))]>,
 223     MTBUFAddr64Table<1>;
 224
 225   def _OFFEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 226   def _IDXEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 227   def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 228
 229   let DisableWQM = 1 in {
 230     def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 231     def _OFFEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 232     def _IDXEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 233     def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 234   }
 235 }
 236
 237 class MTBUF_Store_Pseudo <string opName,
 238                           int addrKind,
 239                           RegisterClass vdataClass,
 240                           list<dag> pattern=[],
 241                           // Workaround bug bz30254
 242                           int addrKindCopy = addrKind,
 243                           RegisterClass vdataClassCopy = vdataClass>
 244   : MTBUF_Pseudo<opName,
 245                  (outs),
 246                  getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 247                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
 248                  pattern>,
 249     MTBUF_SetupAddr<addrKindCopy> {
 250   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 251   let mayLoad = 0;
 252   let mayStore = 1;
 253 }
 254
 255 multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 256                                ValueType store_vt = i32,
 257                                SDPatternOperator st = null_frag> {
 258
 259   def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 260     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 261                                        i16:$offset, i8:$dfmt, i8:$nfmt, i1:$glc,
 262                                        i1:$slc, i1:$tfe))]>,
 263     MTBUFAddr64Table<0>;
 264
 265   def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 266     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 267                                        i16:$offset, i8:$dfmt, i8:$nfmt, i1:$glc,
 268                                        i1:$slc, i1:$tfe))]>,
 269     MTBUFAddr64Table<1>;
 270
 271   def _OFFEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 272   def _IDXEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 273   def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 274
 275   let DisableWQM = 1 in {
 276     def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 277     def _OFFEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 278     def _IDXEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 279     def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 280   }
 281 }
 282
 283
 284 //===----------------------------------------------------------------------===//
 285 // MUBUF classes
 286 //===----------------------------------------------------------------------===//
 287
 288 class MUBUF_Pseudo <string opName, dag outs, dag ins,
 289                     string asmOps, list<dag> pattern=[]> :
 290   InstSI<outs, ins, "", pattern>,
 291   SIMCInstr<opName, SIEncodingFamily.NONE> {
 292
 293   let isPseudo = 1;
 294   let isCodeGenOnly = 1;
 295   let Size = 8;
 296   let UseNamedOperandTable = 1;
 297
 298   string Mnemonic = opName;
 299   string AsmOperands = asmOps;
 300
 301   let VM_CNT = 1;
 302   let EXP_CNT = 1;
 303   let MUBUF = 1;
 304   let Uses = [EXEC];
 305   let hasSideEffects = 0;
 306   let SchedRW = [WriteVMEM];
 307
 308   let AsmMatchConverter = "cvtMubuf";
 309
 310   bits<1> offen       = 0;
 311   bits<1> idxen       = 0;
 312   bits<1> addr64      = 0;
 313   bits<1> has_vdata   = 1;
 314   bits<1> has_vaddr   = 1;
 315   bits<1> has_glc     = 1;
 316   bits<1> glc_value   = 0; // the value for glc if no such operand
 317   bits<1> has_srsrc   = 1;
 318   bits<1> has_soffset = 1;
 319   bits<1> has_offset  = 1;
 320   bits<1> has_slc     = 1;
 321   bits<1> has_tfe     = 1;
 322 }
 323
 324 class MUBUF_Real <bits<7> op, MUBUF_Pseudo ps> :
 325   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 326
 327   let isPseudo = 0;
 328   let isCodeGenOnly = 0;
 329
 330   // copy relevant pseudo op flags
 331   let SubtargetPredicate = ps.SubtargetPredicate;
 332   let AsmMatchConverter  = ps.AsmMatchConverter;
 333   let Constraints        = ps.Constraints;
 334   let DisableEncoding    = ps.DisableEncoding;
 335   let TSFlags            = ps.TSFlags;
 336
 337   bits<12> offset;
 338   bits<1>  glc;
 339   bits<1>  lds = 0;
 340   bits<8>  vaddr;
 341   bits<8>  vdata;
 342   bits<7>  srsrc;
 343   bits<1>  slc;
 344   bits<1>  tfe;
 345   bits<8>  soffset;
 346 }
 347
 348
 349 // For cache invalidation instructions.
 350 class MUBUF_Invalidate <string opName, SDPatternOperator node> :
 351   MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {
 352
 353   let AsmMatchConverter = "";
 354
 355   let hasSideEffects = 1;
 356   let mayStore = 1;
 357
 358   // Set everything to 0.
 359   let offen       = 0;
 360   let idxen       = 0;
 361   let addr64      = 0;
 362   let has_vdata   = 0;
 363   let has_vaddr   = 0;
 364   let has_glc     = 0;
 365   let glc_value   = 0;
 366   let has_srsrc   = 0;
 367   let has_soffset = 0;
 368   let has_offset  = 0;
 369   let has_slc     = 0;
 370   let has_tfe     = 0;
 371 }
 372
 373 class getMUBUFInsDA<list<RegisterClass> vdataList,
 374                     list<RegisterClass> vaddrList=[]> {
 375   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 376   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 377   dag InsNoData = !if(!empty(vaddrList),
 378     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 379          offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe),
 380     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 381          offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe)
 382   );
 383   dag InsData = !if(!empty(vaddrList),
 384     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 385          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe),
 386     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 387          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe)
 388   );
 389   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 390 }
 391
 392 class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
 393   dag ret =
 394     !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList>.ret,
 395     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMUBUFInsDA<vdataList, [VGPR_32]>.ret,
 396     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMUBUFInsDA<vdataList, [VGPR_32]>.ret,
 397     !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64]>.ret,
 398     !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64]>.ret,
 399     (ins))))));
 400 }
 401
 402 class getMUBUFAsmOps<int addrKind> {
 403   string Pfx =
 404     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",
 405     !if(!eq(addrKind, BUFAddrKind.OffEn),  "$vaddr, $srsrc, $soffset offen",
 406     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "$vaddr, $srsrc, $soffset idxen",
 407     !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",
 408     !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",
 409     "")))));
 410   string ret = Pfx # "$offset";
 411 }
 412
 413 class MUBUF_SetupAddr<int addrKind> {
 414   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 415                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 416
 417   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 418                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 419
 420   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 421
 422   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 423 }
 424
 425 class MUBUF_Load_Pseudo <string opName,
 426                          int addrKind,
 427                          RegisterClass vdataClass,
 428                          bit HasTiedDest = 0,
 429                          list<dag> pattern=[],
 430                          // Workaround bug bz30254
 431                          int addrKindCopy = addrKind>
 432   : MUBUF_Pseudo<opName,
 433                  (outs vdataClass:$vdata),
 434                  !con(getMUBUFIns<addrKindCopy>.ret, !if(HasTiedDest, (ins vdataClass:$vdata_in), (ins))),
 435                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
 436                  pattern>,
 437     MUBUF_SetupAddr<addrKindCopy> {
 438   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 439   let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", "");
 440   let mayLoad = 1;
 441   let mayStore = 0;
 442   let maybeAtomic = 1;
 443 }
 444
 445 // FIXME: tfe can't be an operand because it requires a separate
 446 // opcode because it needs an N+1 register class dest register.
 447 multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 448                               ValueType load_vt = i32,
 449                               SDPatternOperator ld = null_frag,
 450                               bit TiedDest = 0> {
 451
 452   def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 453     TiedDest,
 454     [(set load_vt:$vdata,
 455      (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>,
 456     MUBUFAddr64Table<0>;
 457
 458   def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 459     TiedDest,
 460     [(set load_vt:$vdata,
 461      (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>,
 462     MUBUFAddr64Table<1>;
 463
 464   def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest>;
 465   def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest>;
 466   def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest>;
 467
 468   let DisableWQM = 1 in {
 469     def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, TiedDest>;
 470     def _OFFEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest>;
 471     def _IDXEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest>;
 472     def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest>;
 473   }
 474 }
 475
 476 class MUBUF_Store_Pseudo <string opName,
 477                           int addrKind,
 478                           RegisterClass vdataClass,
 479                           list<dag> pattern=[],
 480                           // Workaround bug bz30254
 481                           int addrKindCopy = addrKind,
 482                           RegisterClass vdataClassCopy = vdataClass>
 483   : MUBUF_Pseudo<opName,
 484                  (outs),
 485                  getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 486                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
 487                  pattern>,
 488     MUBUF_SetupAddr<addrKindCopy> {
 489   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 490   let mayLoad = 0;
 491   let mayStore = 1;
 492   let maybeAtomic = 1;
 493 }
 494
 495 multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 496                                ValueType store_vt = i32,
 497                                SDPatternOperator st = null_frag> {
 498
 499   def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 500     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 501                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>,
 502     MUBUFAddr64Table<0>;
 503
 504   def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 505     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 506                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>,
 507     MUBUFAddr64Table<1>;
 508
 509   def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 510   def _IDXEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 511   def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 512
 513   let DisableWQM = 1 in {
 514     def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 515     def _OFFEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 516     def _IDXEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 517     def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 518   }
 519 }
 520
 521
 522 class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
 523                           list<RegisterClass> vaddrList=[]> {
 524   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 525   dag ret = !if(vdata_in,
 526     !if(!empty(vaddrList),
 527       (ins vdataClass:$vdata_in,
 528            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc),
 529       (ins vdataClass:$vdata_in, vaddrClass:$vaddr,
 530            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc)
 531     ),
 532     !if(!empty(vaddrList),
 533       (ins vdataClass:$vdata,
 534            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc),
 535       (ins vdataClass:$vdata, vaddrClass:$vaddr,
 536            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc)
 537   ));
 538 }
 539
 540 class getMUBUFAtomicIns<int addrKind,
 541                         RegisterClass vdataClass,
 542                         bit vdata_in,
 543                         // Workaround bug bz30254
 544                         RegisterClass vdataClassCopy=vdataClass> {
 545   dag ret =
 546     !if(!eq(addrKind, BUFAddrKind.Offset),
 547             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret,
 548     !if(!eq(addrKind, BUFAddrKind.OffEn),
 549             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 550     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 551             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 552     !if(!eq(addrKind, BUFAddrKind.BothEn),
 553             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 554     !if(!eq(addrKind, BUFAddrKind.Addr64),
 555             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 556     (ins))))));
 557 }
 558
 559 class MUBUF_Atomic_Pseudo<string opName,
 560                           int addrKind,
 561                           dag outs,
 562                           dag ins,
 563                           string asmOps,
 564                           list<dag> pattern=[],
 565                           // Workaround bug bz30254
 566                           int addrKindCopy = addrKind>
 567   : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,
 568     MUBUF_SetupAddr<addrKindCopy> {
 569   let mayStore = 1;
 570   let mayLoad = 1;
 571   let hasPostISelHook = 1;
 572   let hasSideEffects = 1;
 573   let DisableWQM = 1;
 574   let has_glc = 0;
 575   let has_tfe = 0;
 576   let maybeAtomic = 1;
 577 }
 578
 579 class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
 580                                RegisterClass vdataClass,
 581                                list<dag> pattern=[],
 582                                // Workaround bug bz30254
 583                                int addrKindCopy = addrKind,
 584                                RegisterClass vdataClassCopy = vdataClass>
 585   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 586                         (outs),
 587                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret,
 588                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc",
 589                         pattern>,
 590     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> {
 591   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 592   let glc_value = 0;
 593   let AsmMatchConverter = "cvtMubufAtomic";
 594 }
 595
 596 class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
 597                              RegisterClass vdataClass,
 598                              list<dag> pattern=[],
 599                              // Workaround bug bz30254
 600                              int addrKindCopy = addrKind,
 601                              RegisterClass vdataClassCopy = vdataClass>
 602   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 603                         (outs vdataClassCopy:$vdata),
 604                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret,
 605                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc",
 606                         pattern>,
 607     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> {
 608   let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret;
 609   let glc_value = 1;
 610   let Constraints = "$vdata = $vdata_in";
 611   let DisableEncoding = "$vdata_in";
 612   let AsmMatchConverter = "cvtMubufAtomicReturn";
 613 }
 614
 615 multiclass MUBUF_Pseudo_Atomics <string opName,
 616                                  RegisterClass vdataClass,
 617                                  ValueType vdataType,
 618                                  SDPatternOperator atomic> {
 619
 620   def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>,
 621                 MUBUFAddr64Table <0>;
 622   def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>,
 623                 MUBUFAddr64Table <1>;
 624   def _OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 625   def _IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 626   def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 627
 628   def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 629     [(set vdataType:$vdata,
 630      (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc),
 631              vdataType:$vdata_in))]>,
 632     MUBUFAddr64Table <0, "_RTN">;
 633
 634   def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 635     [(set vdataType:$vdata,
 636      (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc),
 637              vdataType:$vdata_in))]>,
 638     MUBUFAddr64Table <1, "_RTN">;
 639
 640   def _OFFEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 641   def _IDXEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 642   def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 643 }
 644
 645
 646 //===----------------------------------------------------------------------===//
 647 // MUBUF Instructions
 648 //===----------------------------------------------------------------------===//
 649
 650 defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads <
 651   "buffer_load_format_x", VGPR_32
 652 >;
 653 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
 654   "buffer_load_format_xy", VReg_64
 655 >;
 656 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <
 657   "buffer_load_format_xyz", VReg_96
 658 >;
 659 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <
 660   "buffer_load_format_xyzw", VReg_128
 661 >;
 662 defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <
 663   "buffer_store_format_x", VGPR_32
 664 >;
 665 defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <
 666   "buffer_store_format_xy", VReg_64
 667 >;
 668 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <
 669   "buffer_store_format_xyz", VReg_96
 670 >;
 671 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <
 672   "buffer_store_format_xyzw", VReg_128
 673 >;
 674 defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads <
 675   "buffer_load_ubyte", VGPR_32, i32, mubuf_az_extloadi8
 676 >;
 677 defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads <
 678   "buffer_load_sbyte", VGPR_32, i32, mubuf_sextloadi8
 679 >;
 680 defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads <
 681   "buffer_load_ushort", VGPR_32, i32, mubuf_az_extloadi16
 682 >;
 683 defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads <
 684   "buffer_load_sshort", VGPR_32, i32, mubuf_sextloadi16
 685 >;
 686 defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads <
 687   "buffer_load_dword", VGPR_32, i32, mubuf_load
 688 >;
 689 defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
 690   "buffer_load_dwordx2", VReg_64, v2i32, mubuf_load
 691 >;
 692 defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
 693   "buffer_load_dwordx3", VReg_96, untyped, mubuf_load
 694 >;
 695 defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
 696   "buffer_load_dwordx4", VReg_128, v4i32, mubuf_load
 697 >;
 698 defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
 699   "buffer_store_byte", VGPR_32, i32, truncstorei8_global
 700 >;
 701 defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
 702   "buffer_store_short", VGPR_32, i32, truncstorei16_global
 703 >;
 704 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
 705   "buffer_store_dword", VGPR_32, i32, store_global
 706 >;
 707 defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
 708   "buffer_store_dwordx2", VReg_64, v2i32, store_global
 709 >;
 710 defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
 711   "buffer_store_dwordx3", VReg_96, untyped, store_global
 712 >;
 713 defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
 714   "buffer_store_dwordx4", VReg_128, v4i32, store_global
 715 >;
 716 defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
 717   "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global
 718 >;
 719 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <
 720   "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag
 721 >;
 722 defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <
 723   "buffer_atomic_add", VGPR_32, i32, atomic_add_global
 724 >;
 725 defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <
 726   "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global
 727 >;
 728 defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <
 729   "buffer_atomic_smin", VGPR_32, i32, atomic_min_global
 730 >;
 731 defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <
 732   "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global
 733 >;
 734 defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <
 735   "buffer_atomic_smax", VGPR_32, i32, atomic_max_global
 736 >;
 737 defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <
 738   "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global
 739 >;
 740 defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <
 741   "buffer_atomic_and", VGPR_32, i32, atomic_and_global
 742 >;
 743 defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <
 744   "buffer_atomic_or", VGPR_32, i32, atomic_or_global
 745 >;
 746 defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <
 747   "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global
 748 >;
 749 defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <
 750   "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global
 751 >;
 752 defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <
 753   "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global
 754 >;
 755 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <
 756   "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global
 757 >;
 758 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <
 759   "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag
 760 >;
 761 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <
 762   "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global
 763 >;
 764 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <
 765   "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global
 766 >;
 767 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <
 768   "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global
 769 >;
 770 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <
 771   "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global
 772 >;
 773 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <
 774   "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global
 775 >;
 776 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <
 777   "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global
 778 >;
 779 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <
 780   "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global
 781 >;
 782 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <
 783   "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global
 784 >;
 785 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <
 786   "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global
 787 >;
 788 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <
 789   "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global
 790 >;
 791 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
 792   "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global
 793 >;
 794
 795 let SubtargetPredicate = isSI in { // isn't on CI & VI
 796 /*
 797 defm BUFFER_ATOMIC_RSUB        : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
 798 defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">;
 799 defm BUFFER_ATOMIC_FMIN        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">;
 800 defm BUFFER_ATOMIC_FMAX        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">;
 801 defm BUFFER_ATOMIC_RSUB_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
 802 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">;
 803 defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">;
 804 defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">;
 805 */
 806
 807 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
 808                                           int_amdgcn_buffer_wbinvl1_sc>;
 809 }
 810
 811 let SubtargetPredicate = HasD16LoadStore in {
 812
 813 defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads <
 814   "buffer_load_ubyte_d16", VGPR_32, i32, null_frag, 1
 815 >;
 816
 817 defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads <
 818   "buffer_load_ubyte_d16_hi", VGPR_32, i32, null_frag, 1
 819 >;
 820
 821 defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads <
 822   "buffer_load_sbyte_d16", VGPR_32, i32, null_frag, 1
 823 >;
 824
 825 defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads <
 826   "buffer_load_sbyte_d16_hi", VGPR_32, i32, null_frag, 1
 827 >;
 828
 829 defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads <
 830   "buffer_load_short_d16", VGPR_32, i32, null_frag, 1
 831 >;
 832
 833 defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads <
 834   "buffer_load_short_d16_hi", VGPR_32, i32, null_frag, 1
 835 >;
 836
 837 defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores <
 838   "buffer_store_byte_d16_hi", VGPR_32, i32
 839 >;
 840
 841 defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores <
 842   "buffer_store_short_d16_hi", VGPR_32, i32
 843 >;
 844
 845 } // End HasD16LoadStore
 846
 847 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
 848                                        int_amdgcn_buffer_wbinvl1>;
 849
 850 //===----------------------------------------------------------------------===//
 851 // MTBUF Instructions
 852 //===----------------------------------------------------------------------===//
 853
 854 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32>;
 855 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xy",    VReg_64>;
 856 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyz",   VReg_128>;
 857 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyzw",  VReg_128>;
 858 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_x",    VGPR_32>;
 859 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy",   VReg_64>;
 860 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz",  VReg_128>;
 861 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>;
 862
 863 let SubtargetPredicate = isCIVI in {
 864
 865 //===----------------------------------------------------------------------===//
 866 // Instruction definitions for CI and newer.
 867 //===----------------------------------------------------------------------===//
 868 // Remaining instructions:
 869 // BUFFER_LOAD_DWORDX3
 870 // BUFFER_STORE_DWORDX3
 871
 872 def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
 873                                            int_amdgcn_buffer_wbinvl1_vol>;
 874
 875 } // End let SubtargetPredicate = isCIVI
 876
 877 //===----------------------------------------------------------------------===//
 878 // MUBUF Patterns
 879 //===----------------------------------------------------------------------===//
 880
 881 //===----------------------------------------------------------------------===//
 882 // buffer_load/store_format patterns
 883 //===----------------------------------------------------------------------===//
 884
 885 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
 886                                   string opcode> {
 887   def : GCNPat<
 888     (vt (name v4i32:$rsrc, 0,
 889               (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 890               imm:$glc, imm:$slc)),
 891     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
 892       (as_i1imm $glc), (as_i1imm $slc), 0)
 893   >;
 894
 895   def : GCNPat<
 896     (vt (name v4i32:$rsrc, i32:$vindex,
 897               (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 898               imm:$glc, imm:$slc)),
 899     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
 900       (as_i1imm $glc), (as_i1imm $slc), 0)
 901   >;
 902
 903   def : GCNPat<
 904     (vt (name v4i32:$rsrc, 0,
 905               (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 906               imm:$glc, imm:$slc)),
 907     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
 908       (as_i1imm $glc), (as_i1imm $slc), 0)
 909   >;
 910
 911   def : GCNPat<
 912     (vt (name v4i32:$rsrc, i32:$vindex,
 913               (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 914               imm:$glc, imm:$slc)),
 915     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
 916       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
 917       $rsrc, $soffset, (as_i16imm $offset),
 918       (as_i1imm $glc), (as_i1imm $slc), 0)
 919   >;
 920 }
 921
 922 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
 923 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
 924 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;
 925 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">;
 926 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">;
 927 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">;
 928
 929 multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
 930                                    string opcode> {
 931   def : GCNPat<
 932     (name vt:$vdata, v4i32:$rsrc, 0,
 933           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 934           imm:$glc, imm:$slc),
 935     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset),
 936                                     (as_i1imm $glc), (as_i1imm $slc), 0)
 937   >;
 938
 939   def : GCNPat<
 940     (name vt:$vdata, v4i32:$rsrc, i32:$vindex,
 941           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 942           imm:$glc, imm:$slc),
 943     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
 944                                    (as_i16imm $offset), (as_i1imm $glc),
 945                                    (as_i1imm $slc), 0)
 946   >;
 947
 948   def : GCNPat<
 949     (name vt:$vdata, v4i32:$rsrc, 0,
 950           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 951           imm:$glc, imm:$slc),
 952     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
 953                                    (as_i16imm $offset), (as_i1imm $glc),
 954                                    (as_i1imm $slc), 0)
 955   >;
 956
 957   def : GCNPat<
 958     (name vt:$vdata, v4i32:$rsrc, i32:$vindex,
 959           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
 960           imm:$glc, imm:$slc),
 961     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
 962       $vdata,
 963       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
 964       $rsrc, $soffset, (as_i16imm $offset),
 965       (as_i1imm $glc), (as_i1imm $slc), 0)
 966   >;
 967 }
 968
 969 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
 970 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
 971 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
 972 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">;
 973 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">;
 974 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">;
 975
 976 //===----------------------------------------------------------------------===//
 977 // buffer_atomic patterns
 978 //===----------------------------------------------------------------------===//
 979
 980 multiclass BufferAtomicPatterns<SDPatternOperator name, string opcode> {
 981   def : GCNPat<
 982     (name i32:$vdata_in, v4i32:$rsrc, 0,
 983           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 984           imm:$slc),
 985     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset,
 986                                         (as_i16imm $offset), (as_i1imm $slc))
 987   >;
 988
 989   def : GCNPat<
 990     (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex,
 991           (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
 992           imm:$slc),
 993     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset,
 994                                        (as_i16imm $offset), (as_i1imm $slc))
 995   >;
 996
 997   def : GCNPat<
 998     (name i32:$vdata_in, v4i32:$rsrc, 0,
 999           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
1000           imm:$slc),
1001     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset,
1002                                        (as_i16imm $offset), (as_i1imm $slc))
1003   >;
1004
1005   def : GCNPat<
1006     (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex,
1007           (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
1008           imm:$slc),
1009     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN)
1010       $vdata_in,
1011       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1012       $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc))
1013   >;
1014 }
1015
1016 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, "BUFFER_ATOMIC_SWAP">;
1017 defm : BufferAtomicPatterns<SIbuffer_atomic_add, "BUFFER_ATOMIC_ADD">;
1018 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, "BUFFER_ATOMIC_SUB">;
1019 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, "BUFFER_ATOMIC_SMIN">;
1020 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, "BUFFER_ATOMIC_UMIN">;
1021 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, "BUFFER_ATOMIC_SMAX">;
1022 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, "BUFFER_ATOMIC_UMAX">;
1023 defm : BufferAtomicPatterns<SIbuffer_atomic_and, "BUFFER_ATOMIC_AND">;
1024 defm : BufferAtomicPatterns<SIbuffer_atomic_or, "BUFFER_ATOMIC_OR">;
1025 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, "BUFFER_ATOMIC_XOR">;
1026
1027 def : GCNPat<
1028   (SIbuffer_atomic_cmpswap
1029       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1030       (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
1031       imm:$slc),
1032   (EXTRACT_SUBREG
1033     (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN
1034       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1035       $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
1036     sub0)
1037 >;
1038
1039 def : GCNPat<
1040   (SIbuffer_atomic_cmpswap
1041       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1042       (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
1043       imm:$slc),
1044   (EXTRACT_SUBREG
1045     (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN
1046       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1047       $vindex, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
1048     sub0)
1049 >;
1050
1051 def : GCNPat<
1052   (SIbuffer_atomic_cmpswap
1053       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1054       (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
1055       imm:$slc),
1056   (EXTRACT_SUBREG
1057     (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN
1058       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1059       $voffset, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
1060     sub0)
1061 >;
1062
1063 def : GCNPat<
1064   (SIbuffer_atomic_cmpswap
1065       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1066       (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
1067       imm:$slc),
1068   (EXTRACT_SUBREG
1069     (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN
1070       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1071       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1072       $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
1073     sub0)
1074 >;
1075
1076
1077 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
1078                               PatFrag constant_ld> : GCNPat <
1079      (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1080                                    i16:$offset, i1:$glc, i1:$slc, i1:$tfe))),
1081      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe)
1082   >;
1083
1084 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1085                                      ValueType vt, PatFrag atomic_ld> {
1086   def : GCNPat <
1087      (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1088                                    i16:$offset, i1:$slc))),
1089      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0)
1090   >;
1091
1092   def : GCNPat <
1093     (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))),
1094     (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0)
1095   >;
1096 }
1097
1098 let SubtargetPredicate = isSICI in {
1099 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
1100 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, az_extloadi8_constant>;
1101 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
1102 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, az_extloadi16_constant>;
1103
1104 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, mubuf_load_atomic>;
1105 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, mubuf_load_atomic>;
1106 } // End SubtargetPredicate = isSICI
1107
1108 multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1109                                PatFrag ld> {
1110
1111   def : GCNPat <
1112     (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1113                           i16:$offset, i1:$glc, i1:$slc, i1:$tfe))),
1114     (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe)
1115   >;
1116 }
1117
1118 let OtherPredicates = [Has16BitInsts] in {
1119
1120 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>;
1121 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_constant>;
1122 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, mubuf_sextloadi8>;
1123 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, mubuf_az_extloadi8>;
1124
1125 defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, mubuf_load>;
1126
1127 } // End OtherPredicates = [Has16BitInsts]
1128
1129 multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen,
1130                                 MUBUF_Pseudo InstrOffset,
1131                                 ValueType vt, PatFrag ld> {
1132   def : GCNPat <
1133     (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1134                                i32:$soffset, u16imm:$offset))),
1135     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
1136   >;
1137
1138   def : GCNPat <
1139     (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))),
1140     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0)
1141   >;
1142 }
1143
1144 // XXX - Is it possible to have a complex pattern in a PatFrag?
1145 multiclass MUBUFScratchLoadPat_Hi16 <MUBUF_Pseudo InstrOffen,
1146                                 MUBUF_Pseudo InstrOffset,
1147                                 ValueType vt, PatFrag ld> {
1148   def : GCNPat <
1149     (build_vector vt:$lo, (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1150                                  i32:$soffset, u16imm:$offset)))),
1151     (v2i16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $lo))
1152   >;
1153
1154   def : GCNPat <
1155     (build_vector f16:$lo, (f16 (bitconvert (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1156                                i32:$soffset, u16imm:$offset)))))),
1157     (v2f16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $lo))
1158   >;
1159
1160
1161   def : GCNPat <
1162     (build_vector vt:$lo, (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset)))),
1163     (v2i16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $lo))
1164   >;
1165
1166   def : GCNPat <
1167     (build_vector f16:$lo, (f16 (bitconvert (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset)))))),
1168     (v2f16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $lo))
1169   >;
1170 }
1171
1172 multiclass MUBUFScratchLoadPat_Lo16 <MUBUF_Pseudo InstrOffen,
1173                                      MUBUF_Pseudo InstrOffset,
1174                                      ValueType vt, PatFrag ld> {
1175   def : GCNPat <
1176     (build_vector (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1177                                              i32:$soffset, u16imm:$offset))),
1178                   (vt (Hi16Elt vt:$hi))),
1179     (v2i16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $hi))
1180   >;
1181
1182   def : GCNPat <
1183     (build_vector (f16 (bitconvert (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1184                                                               i32:$soffset, u16imm:$offset))))),
1185                   (f16 (Hi16Elt f16:$hi))),
1186     (v2f16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $hi))
1187   >;
1188
1189   def : GCNPat <
1190     (build_vector (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))),
1191                   (vt (Hi16Elt vt:$hi))),
1192     (v2i16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $hi))
1193   >;
1194
1195   def : GCNPat <
1196     (build_vector (f16 (bitconvert (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))))),
1197                   (f16 (Hi16Elt f16:$hi))),
1198     (v2f16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $hi))
1199   >;
1200 }
1201
1202 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>;
1203 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, az_extloadi8_private>;
1204 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>;
1205 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_private>;
1206 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>;
1207 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, az_extloadi16_private>;
1208 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>;
1209 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>;
1210 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>;
1211 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>;
1212
1213 let OtherPredicates = [HasD16LoadStore] in {
1214 defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, i16, load_private>;
1215 defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, i16, az_extloadi8_private>;
1216 defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, i16, sextloadi8_private>;
1217
1218 defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, i16, load_private>;
1219 defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, i16, az_extloadi8_private>;
1220 defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, i16, sextloadi8_private>;
1221 }
1222
1223 // BUFFER_LOAD_DWORD*, addr64=0
1224 multiclass MUBUF_Load_Dword <ValueType vt,
1225                              MUBUF_Pseudo offset,
1226                              MUBUF_Pseudo offen,
1227                              MUBUF_Pseudo idxen,
1228                              MUBUF_Pseudo bothen> {
1229
1230   def : GCNPat <
1231     (vt (int_SI_buffer_load_dword v4i32:$rsrc, (i32 imm), i32:$soffset,
1232                                   imm:$offset, 0, 0, imm:$glc, imm:$slc,
1233                                   imm:$tfe)),
1234     (offset $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc),
1235             (as_i1imm $slc), (as_i1imm $tfe))
1236   >;
1237
1238   def : GCNPat <
1239     (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset,
1240                                   imm:$offset, 1, 0, imm:$glc, imm:$slc,
1241                                   imm:$tfe)),
1242     (offen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc),
1243            (as_i1imm $tfe))
1244   >;
1245
1246   def : GCNPat <
1247     (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset,
1248                                   imm:$offset, 0, 1, imm:$glc, imm:$slc,
1249                                   imm:$tfe)),
1250     (idxen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc),
1251            (as_i1imm $slc), (as_i1imm $tfe))
1252   >;
1253
1254   def : GCNPat <
1255     (vt (int_SI_buffer_load_dword v4i32:$rsrc, v2i32:$vaddr, i32:$soffset,
1256                                   imm:$offset, 1, 1, imm:$glc, imm:$slc,
1257                                   imm:$tfe)),
1258     (bothen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc),
1259             (as_i1imm $tfe))
1260   >;
1261 }
1262
1263 defm : MUBUF_Load_Dword <i32, BUFFER_LOAD_DWORD_OFFSET, BUFFER_LOAD_DWORD_OFFEN,
1264                          BUFFER_LOAD_DWORD_IDXEN, BUFFER_LOAD_DWORD_BOTHEN>;
1265 defm : MUBUF_Load_Dword <v2i32, BUFFER_LOAD_DWORDX2_OFFSET, BUFFER_LOAD_DWORDX2_OFFEN,
1266                          BUFFER_LOAD_DWORDX2_IDXEN, BUFFER_LOAD_DWORDX2_BOTHEN>;
1267 defm : MUBUF_Load_Dword <v4i32, BUFFER_LOAD_DWORDX4_OFFSET, BUFFER_LOAD_DWORDX4_OFFEN,
1268                          BUFFER_LOAD_DWORDX4_IDXEN, BUFFER_LOAD_DWORDX4_BOTHEN>;
1269
1270 multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1271                                       ValueType vt, PatFrag atomic_st> {
1272   // Store follows atomic op convention so address is forst
1273   def : GCNPat <
1274      (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1275                                    i16:$offset, i1:$slc), vt:$val),
1276      (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0)
1277   >;
1278
1279   def : GCNPat <
1280     (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
1281     (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0)
1282   >;
1283 }
1284 let SubtargetPredicate = isSICI in {
1285 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>;
1286 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>;
1287 } // End Predicates = isSICI
1288
1289
1290 multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1291                                PatFrag st> {
1292
1293   def : GCNPat <
1294     (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1295                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe)),
1296     (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe)
1297   >;
1298 }
1299
1300 defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>;
1301 defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>;
1302
1303 multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen,
1304                                  MUBUF_Pseudo InstrOffset,
1305                                  ValueType vt, PatFrag st> {
1306   def : GCNPat <
1307     (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1308                                       i32:$soffset, u16imm:$offset)),
1309     (InstrOffen $value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
1310   >;
1311
1312   def : GCNPat <
1313     (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,
1314                                        u16imm:$offset)),
1315     (InstrOffset $value, $srsrc, $soffset, $offset, 0, 0, 0)
1316   >;
1317 }
1318
1319 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>;
1320 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>;
1321 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>;
1322 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>;
1323 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>;
1324 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private>;
1325 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private>;
1326
1327
1328 let OtherPredicates = [HasD16LoadStore] in {
1329  // Hiding the extract high pattern in the PatFrag seems to not
1330  // automatically increase the complexity.
1331 let AddedComplexity = 1 in {
1332 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>;
1333 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>;
1334 }
1335 }
1336
1337 //===----------------------------------------------------------------------===//
1338 // MTBUF Patterns
1339 //===----------------------------------------------------------------------===//
1340
1341 //===----------------------------------------------------------------------===//
1342 // tbuffer_load/store_format patterns
1343 //===----------------------------------------------------------------------===//
1344
1345 multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1346                                   string opcode> {
1347   def : GCNPat<
1348     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1349               imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)),
1350     (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
1351       (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1352   >;
1353
1354   def : GCNPat<
1355     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1356               imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)),
1357     (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
1358       (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1359   >;
1360
1361   def : GCNPat<
1362     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1363               imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)),
1364     (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
1365       (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1366   >;
1367
1368   def : GCNPat<
1369     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset,
1370               imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)),
1371     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN)
1372       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1373       $rsrc, $soffset, (as_i16imm $offset),
1374       (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1375   >;
1376 }
1377
1378 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32,   "TBUFFER_LOAD_FORMAT_X">;
1379 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;
1380 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">;
1381 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32,   "TBUFFER_LOAD_FORMAT_X">;
1382 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;
1383 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;
1384
1385 multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1386                                    string opcode> {
1387   def : GCNPat<
1388     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1389           imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc),
1390     (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset,
1391                                 (as_i16imm $offset), (as_i8imm $dfmt),
1392                                 (as_i8imm $nfmt), (as_i1imm $glc),
1393                                 (as_i1imm $slc), 0)
1394   >;
1395
1396   def : GCNPat<
1397     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1398           imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc),
1399     (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
1400                                    (as_i16imm $offset), (as_i8imm $dfmt),
1401                                    (as_i8imm $nfmt), (as_i1imm $glc),
1402                                    (as_i1imm $slc), 0)
1403   >;
1404
1405   def : GCNPat<
1406     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1407           imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc),
1408     (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
1409                                    (as_i16imm $offset), (as_i8imm $dfmt),
1410                                    (as_i8imm $nfmt), (as_i1imm $glc),
1411                                    (as_i1imm $slc), 0)
1412   >;
1413
1414   def : GCNPat<
1415     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset,
1416           imm:$offset, imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc),
1417     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact)
1418       $vdata,
1419       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1420       $rsrc, $soffset, (as_i16imm $offset),
1421       (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1422   >;
1423 }
1424
1425 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32,   "TBUFFER_STORE_FORMAT_X">;
1426 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;
1427 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4i32, "TBUFFER_STORE_FORMAT_XYZ">;
1428 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">;
1429 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32,   "TBUFFER_STORE_FORMAT_X">;
1430 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;
1431 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4f32, "TBUFFER_STORE_FORMAT_XYZ">;
1432 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;
1433
1434 //===----------------------------------------------------------------------===//
1435 // Target instructions, move to the appropriate target TD file
1436 //===----------------------------------------------------------------------===//
1437
1438 //===----------------------------------------------------------------------===//
1439 // SI
1440 //===----------------------------------------------------------------------===//
1441
1442 class MUBUF_Real_si <bits<7> op, MUBUF_Pseudo ps> :
1443   MUBUF_Real<op, ps>,
1444   Enc64,
1445   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> {
1446   let AssemblerPredicate=isSICI;
1447   let DecoderNamespace="SICI";
1448
1449   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1450   let Inst{12}    = ps.offen;
1451   let Inst{13}    = ps.idxen;
1452   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1453   let Inst{15}    = ps.addr64;
1454   let Inst{16}    = lds;
1455   let Inst{24-18} = op;
1456   let Inst{31-26} = 0x38; //encoding
1457   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1458   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1459   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1460   let Inst{54}    = !if(ps.has_slc, slc, ?);
1461   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1462   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1463 }
1464
1465 multiclass MUBUF_Real_AllAddr_si<bits<7> op> {
1466   def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1467   def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>;
1468   def _OFFEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1469   def _IDXEN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1470   def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1471 }
1472
1473 multiclass MUBUF_Real_Atomic_si<bits<7> op> : MUBUF_Real_AllAddr_si<op> {
1474   def _OFFSET_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1475   def _ADDR64_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>;
1476   def _OFFEN_RTN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1477   def _IDXEN_RTN_si  : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1478   def _BOTHEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1479 }
1480
1481 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_si <0x00>;
1482 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_si <0x01>;
1483 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_si <0x02>;
1484 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_si <0x03>;
1485 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_si <0x04>;
1486 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_si <0x05>;
1487 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_si <0x06>;
1488 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_si <0x07>;
1489 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_si <0x08>;
1490 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_si <0x09>;
1491 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_si <0x0a>;
1492 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_si <0x0b>;
1493 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_si <0x0c>;
1494 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_si <0x0d>;
1495 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_si <0x0e>;
1496 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_si <0x0f>;
1497 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_si <0x18>;
1498 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_si <0x1a>;
1499 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_si <0x1c>;
1500 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_si <0x1d>;
1501 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_si <0x1e>;
1502 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_si <0x1f>;
1503
1504 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_si <0x30>;
1505 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_si <0x31>;
1506 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_si <0x32>;
1507 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_si <0x33>;
1508 //defm BUFFER_ATOMIC_RSUB         : MUBUF_Real_Atomic_si <0x34>;    // isn't on CI & VI
1509 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_si <0x35>;
1510 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_si <0x36>;
1511 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_si <0x37>;
1512 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_si <0x38>;
1513 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_si <0x39>;
1514 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_si <0x3a>;
1515 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_si <0x3b>;
1516 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_si <0x3c>;
1517 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_si <0x3d>;
1518
1519 //defm BUFFER_ATOMIC_FCMPSWAP     : MUBUF_Real_Atomic_si <0x3e>;    // isn't on VI
1520 //defm BUFFER_ATOMIC_FMIN         : MUBUF_Real_Atomic_si <0x3f>;    // isn't on VI
1521 //defm BUFFER_ATOMIC_FMAX         : MUBUF_Real_Atomic_si <0x40>;    // isn't on VI
1522 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_si <0x50>;
1523 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_si <0x51>;
1524 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_si <0x52>;
1525 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_si <0x53>;
1526 //defm BUFFER_ATOMIC_RSUB_X2      : MUBUF_Real_Atomic_si <0x54>;    // isn't on CI & VI
1527 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_si <0x55>;
1528 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_si <0x56>;
1529 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_si <0x57>;
1530 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_si <0x58>;
1531 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_si <0x59>;
1532 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_si <0x5a>;
1533 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_si <0x5b>;
1534 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_si <0x5c>;
1535 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_si <0x5d>;
1536 // FIXME: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on CI.
1537 //defm BUFFER_ATOMIC_FCMPSWAP_X2  : MUBUF_Real_Atomic_si <0x5e">;   // isn't on VI
1538 //defm BUFFER_ATOMIC_FMIN_X2      : MUBUF_Real_Atomic_si <0x5f>;    // isn't on VI
1539 //defm BUFFER_ATOMIC_FMAX_X2      : MUBUF_Real_Atomic_si <0x60>;    // isn't on VI
1540
1541 def BUFFER_WBINVL1_SC_si        : MUBUF_Real_si <0x70, BUFFER_WBINVL1_SC>;
1542 def BUFFER_WBINVL1_si           : MUBUF_Real_si <0x71, BUFFER_WBINVL1>;
1543
1544 class MTBUF_Real_si <bits<3> op, MTBUF_Pseudo ps> :
1545   MTBUF_Real<ps>,
1546   Enc64,
1547   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> {
1548   let AssemblerPredicate=isSICI;
1549   let DecoderNamespace="SICI";
1550
1551   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1552   let Inst{12}    = ps.offen;
1553   let Inst{13}    = ps.idxen;
1554   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1555   let Inst{15}    = ps.addr64;
1556   let Inst{18-16} = op;
1557   let Inst{22-19} = !if(ps.has_dfmt, dfmt, ps.dfmt_value);
1558   let Inst{25-23} = !if(ps.has_nfmt, nfmt, ps.nfmt_value);
1559   let Inst{31-26} = 0x3a; //encoding
1560   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1561   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1562   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1563   let Inst{54}    = !if(ps.has_slc, slc, ?);
1564   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1565   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1566 }
1567
1568 multiclass MTBUF_Real_AllAddr_si<bits<3> op> {
1569   def _OFFSET_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
1570   def _ADDR64_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>;
1571   def _OFFEN_si  : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
1572   def _IDXEN_si  : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
1573   def _BOTHEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
1574 }
1575
1576 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_si <0>;
1577 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_si <1>;
1578 //defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_si <2>;
1579 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_si <3>;
1580 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_si <4>;
1581 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_si <5>;
1582 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_si <6>;
1583 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_si <7>;
1584
1585 //===----------------------------------------------------------------------===//
1586 // CI
1587 //===----------------------------------------------------------------------===//
1588
1589 class MUBUF_Real_ci <bits<7> op, MUBUF_Pseudo ps> :
1590   MUBUF_Real_si<op, ps> {
1591   let AssemblerPredicate=isCIOnly;
1592   let DecoderNamespace="CI";
1593 }
1594
1595 def BUFFER_WBINVL1_VOL_ci : MUBUF_Real_ci <0x70, BUFFER_WBINVL1_VOL>;
1596
1597
1598 //===----------------------------------------------------------------------===//
1599 // VI
1600 //===----------------------------------------------------------------------===//
1601
1602 class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> :
1603   MUBUF_Real<op, ps>,
1604   Enc64,
1605   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
1606   let AssemblerPredicate=isVI;
1607   let DecoderNamespace="VI";
1608
1609   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1610   let Inst{12}    = ps.offen;
1611   let Inst{13}    = ps.idxen;
1612   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1613   let Inst{16}    = lds;
1614   let Inst{17}    = !if(ps.has_slc, slc, ?);
1615   let Inst{24-18} = op;
1616   let Inst{31-26} = 0x38; //encoding
1617   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1618   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1619   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1620   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1621   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1622 }
1623
1624 multiclass MUBUF_Real_AllAddr_vi<bits<7> op> {
1625   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1626   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1627   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1628   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1629 }
1630
1631 multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
1632   MUBUF_Real_AllAddr_vi<op> {
1633   def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1634   def _OFFEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1635   def _IDXEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1636   def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1637 }
1638
1639 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_vi <0x00>;
1640 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_vi <0x01>;
1641 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_vi <0x02>;
1642 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_vi <0x03>;
1643 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_vi <0x04>;
1644 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_vi <0x05>;
1645 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_vi <0x06>;
1646 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_vi <0x07>;
1647 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_vi <0x10>;
1648 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_vi <0x11>;
1649 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_vi <0x12>;
1650 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_vi <0x13>;
1651 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_vi <0x14>;
1652 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_vi <0x15>;
1653 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_vi <0x16>;
1654 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_vi <0x17>;
1655 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_vi <0x18>;
1656 defm BUFFER_STORE_BYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x19>;
1657 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_vi <0x1a>;
1658 defm BUFFER_STORE_SHORT_D16_HI  : MUBUF_Real_AllAddr_vi <0x1b>;
1659 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_vi <0x1c>;
1660 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_vi <0x1d>;
1661 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_vi <0x1e>;
1662 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_vi <0x1f>;
1663
1664 defm BUFFER_LOAD_UBYTE_D16      : MUBUF_Real_AllAddr_vi <0x20>;
1665 defm BUFFER_LOAD_UBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x21>;
1666 defm BUFFER_LOAD_SBYTE_D16      : MUBUF_Real_AllAddr_vi <0x22>;
1667 defm BUFFER_LOAD_SBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x23>;
1668 defm BUFFER_LOAD_SHORT_D16      : MUBUF_Real_AllAddr_vi <0x24>;
1669 defm BUFFER_LOAD_SHORT_D16_HI   : MUBUF_Real_AllAddr_vi <0x25>;
1670
1671 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_vi <0x40>;
1672 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_vi <0x41>;
1673 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_vi <0x42>;
1674 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_vi <0x43>;
1675 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_vi <0x44>;
1676 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_vi <0x45>;
1677 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_vi <0x46>;
1678 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_vi <0x47>;
1679 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_vi <0x48>;
1680 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_vi <0x49>;
1681 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_vi <0x4a>;
1682 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_vi <0x4b>;
1683 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_vi <0x4c>;
1684
1685 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_vi <0x60>;
1686 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_vi <0x61>;
1687 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_vi <0x62>;
1688 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_vi <0x63>;
1689 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_vi <0x64>;
1690 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_vi <0x65>;
1691 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_vi <0x66>;
1692 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_vi <0x67>;
1693 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_vi <0x68>;
1694 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_vi <0x69>;
1695 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_vi <0x6a>;
1696 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_vi <0x6b>;
1697 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_vi <0x6c>;
1698
1699 def BUFFER_WBINVL1_vi           : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>;
1700 def BUFFER_WBINVL1_VOL_vi       : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>;
1701
1702 class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
1703   MTBUF_Real<ps>,
1704   Enc64,
1705   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
1706   let AssemblerPredicate=isVI;
1707   let DecoderNamespace="VI";
1708
1709   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1710   let Inst{12}    = ps.offen;
1711   let Inst{13}    = ps.idxen;
1712   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1713   let Inst{18-15} = op;
1714   let Inst{22-19} = !if(ps.has_dfmt, dfmt, ps.dfmt_value);
1715   let Inst{25-23} = !if(ps.has_nfmt, nfmt, ps.nfmt_value);
1716   let Inst{31-26} = 0x3a; //encoding
1717   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1718   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1719   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1720   let Inst{54}    = !if(ps.has_slc, slc, ?);
1721   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1722   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1723 }
1724
1725 multiclass MTBUF_Real_AllAddr_vi<bits<4> op> {
1726   def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
1727   def _OFFEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
1728   def _IDXEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
1729   def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
1730 }
1731
1732 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_vi <0>;
1733 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_vi <1>;
1734 //defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_vi <2>;
1735 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_vi <3>;
1736 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_vi <4>;
1737 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_vi <5>;
1738 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_vi <6>;
1739 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <7>;