contrib/llvm-project/llvm/lib/Target/AMDGPU/BUFInstructions.td

   1 //===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">;
  10 def MUBUFAddr64 : ComplexPattern<i64, 8, "SelectMUBUFAddr64">;
  11 def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">;
  12
  13 def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>;
  14 def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>;
  15
  16 def MUBUFOffset : ComplexPattern<i64, 7, "SelectMUBUFOffset">;
  17 def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
  18 def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">;
  19
  20 def BUFAddrKind {
  21   int Offset = 0;
  22   int OffEn  = 1;
  23   int IdxEn  = 2;
  24   int BothEn = 3;
  25   int Addr64 = 4;
  26 }
  27
  28 class getAddrName<int addrKind> {
  29   string ret =
  30     !if(!eq(addrKind, BUFAddrKind.Offset), "offset",
  31     !if(!eq(addrKind, BUFAddrKind.OffEn),  "offen",
  32     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "idxen",
  33     !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",
  34     !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",
  35     "")))));
  36 }
  37
  38 class MUBUFAddr64Table <bit is_addr64, string Name> {
  39   bit IsAddr64 = is_addr64;
  40   string OpName = Name;
  41 }
  42
  43 class MUBUFLdsTable <bit is_lds, string Name> {
  44   bit IsLds = is_lds;
  45   string OpName = Name;
  46 }
  47
  48 class MTBUFAddr64Table <bit is_addr64, string Name> {
  49   bit IsAddr64 = is_addr64;
  50   string OpName = Name;
  51 }
  52
  53 //===----------------------------------------------------------------------===//
  54 // MTBUF classes
  55 //===----------------------------------------------------------------------===//
  56
  57 class MTBUF_Pseudo <string opName, dag outs, dag ins,
  58                     string asmOps, list<dag> pattern=[]> :
  59   InstSI<outs, ins, "", pattern>,
  60   SIMCInstr<opName, SIEncodingFamily.NONE> {
  61
  62   let isPseudo = 1;
  63   let isCodeGenOnly = 1;
  64   let Size = 8;
  65   let UseNamedOperandTable = 1;
  66
  67   string Mnemonic = opName;
  68   string AsmOperands = asmOps;
  69
  70   let VM_CNT = 1;
  71   let EXP_CNT = 1;
  72   let MTBUF = 1;
  73   let Uses = [EXEC];
  74   let hasSideEffects = 0;
  75   let SchedRW = [WriteVMEM];
  76
  77   let AsmMatchConverter = "cvtMtbuf";
  78
  79   bits<1> offen       = 0;
  80   bits<1> idxen       = 0;
  81   bits<1> addr64      = 0;
  82   bits<1> has_vdata   = 1;
  83   bits<1> has_vaddr   = 1;
  84   bits<1> has_glc     = 1;
  85   bits<1> has_dlc     = 1;
  86   bits<1> glc_value   = 0; // the value for glc if no such operand
  87   bits<1> dlc_value   = 0; // the value for dlc if no such operand
  88   bits<1> has_srsrc   = 1;
  89   bits<1> has_soffset = 1;
  90   bits<1> has_offset  = 1;
  91   bits<1> has_slc     = 1;
  92   bits<1> has_tfe     = 1;
  93 }
  94
  95 class MTBUF_Real <MTBUF_Pseudo ps> :
  96   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
  97
  98   let isPseudo = 0;
  99   let isCodeGenOnly = 0;
 100
 101   // copy relevant pseudo op flags
 102   let SubtargetPredicate = ps.SubtargetPredicate;
 103   let AsmMatchConverter  = ps.AsmMatchConverter;
 104   let Constraints        = ps.Constraints;
 105   let DisableEncoding    = ps.DisableEncoding;
 106   let TSFlags            = ps.TSFlags;
 107
 108   bits<12> offset;
 109   bits<1>  glc;
 110   bits<1>  dlc;
 111   bits<7>  format;
 112   bits<8>  vaddr;
 113   bits<8>  vdata;
 114   bits<7>  srsrc;
 115   bits<1>  slc;
 116   bits<1>  tfe;
 117   bits<8>  soffset;
 118
 119   bits<4> dfmt = format{3-0};
 120   bits<3> nfmt = format{6-4};
 121 }
 122
 123 class getMTBUFInsDA<list<RegisterClass> vdataList,
 124                     list<RegisterClass> vaddrList=[]> {
 125   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 126   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 127   dag InsNoData = !if(!empty(vaddrList),
 128     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 129          offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc),
 130     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 131          offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc)
 132   );
 133   dag InsData = !if(!empty(vaddrList),
 134     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 135          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
 136          SLC:$slc, TFE:$tfe, DLC:$dlc),
 137     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 138          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
 139          SLC:$slc, TFE:$tfe, DLC:$dlc)
 140   );
 141   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 142 }
 143
 144 class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
 145   dag ret =
 146     !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret,
 147     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 148     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 149     !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 150     !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 151     (ins))))));
 152 }
 153
 154 class getMTBUFAsmOps<int addrKind> {
 155   string Pfx =
 156     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $format, $soffset",
 157     !if(!eq(addrKind, BUFAddrKind.OffEn),
 158             "$vaddr, $srsrc, $format, $soffset offen",
 159     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 160             "$vaddr, $srsrc, $format, $soffset idxen",
 161     !if(!eq(addrKind, BUFAddrKind.BothEn),
 162             "$vaddr, $srsrc, $format, $soffset idxen offen",
 163     !if(!eq(addrKind, BUFAddrKind.Addr64),
 164             "$vaddr, $srsrc, $format, $soffset addr64",
 165     "")))));
 166   string ret = Pfx # "$offset";
 167 }
 168
 169 class MTBUF_SetupAddr<int addrKind> {
 170   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 171                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 172
 173   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 174                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 175
 176   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 177
 178   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 179 }
 180
 181 class MTBUF_Load_Pseudo <string opName,
 182                          int addrKind,
 183                          RegisterClass vdataClass,
 184                          list<dag> pattern=[],
 185                          // Workaround bug bz30254
 186                          int addrKindCopy = addrKind>
 187   : MTBUF_Pseudo<opName,
 188                  (outs vdataClass:$vdata),
 189                  getMTBUFIns<addrKindCopy>.ret,
 190                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc",
 191                  pattern>,
 192     MTBUF_SetupAddr<addrKindCopy> {
 193   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 194   let mayLoad = 1;
 195   let mayStore = 0;
 196 }
 197
 198 multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 199                               ValueType load_vt = i32,
 200                               SDPatternOperator ld = null_frag> {
 201
 202   def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 203     [(set load_vt:$vdata,
 204      (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format,
 205                       i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)))]>,
 206     MTBUFAddr64Table<0, NAME>;
 207
 208   def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 209     [(set load_vt:$vdata,
 210      (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset,
 211                       i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)))]>,
 212     MTBUFAddr64Table<1, NAME>;
 213
 214   def _OFFEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 215   def _IDXEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 216   def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 217
 218   let DisableWQM = 1 in {
 219     def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 220     def _OFFEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 221     def _IDXEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 222     def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 223   }
 224 }
 225
 226 class MTBUF_Store_Pseudo <string opName,
 227                           int addrKind,
 228                           RegisterClass vdataClass,
 229                           list<dag> pattern=[],
 230                           // Workaround bug bz30254
 231                           int addrKindCopy = addrKind,
 232                           RegisterClass vdataClassCopy = vdataClass>
 233   : MTBUF_Pseudo<opName,
 234                  (outs),
 235                  getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 236                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc",
 237                  pattern>,
 238     MTBUF_SetupAddr<addrKindCopy> {
 239   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 240   let mayLoad = 0;
 241   let mayStore = 1;
 242 }
 243
 244 multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 245                                ValueType store_vt = i32,
 246                                SDPatternOperator st = null_frag> {
 247
 248   def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 249     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 250                                        i16:$offset, i8:$format, i1:$glc,
 251                                        i1:$slc, i1:$tfe, i1:$dlc))]>,
 252     MTBUFAddr64Table<0, NAME>;
 253
 254   def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 255     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 256                                        i16:$offset, i8:$format, i1:$glc,
 257                                        i1:$slc, i1:$tfe, i1:$dlc))]>,
 258     MTBUFAddr64Table<1, NAME>;
 259
 260   def _OFFEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 261   def _IDXEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 262   def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 263
 264   let DisableWQM = 1 in {
 265     def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 266     def _OFFEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 267     def _IDXEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 268     def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 269   }
 270 }
 271
 272
 273 //===----------------------------------------------------------------------===//
 274 // MUBUF classes
 275 //===----------------------------------------------------------------------===//
 276
 277 class MUBUFGetBaseOpcode<string Op> {
 278   string ret = !subst("DWORDX2", "DWORD",
 279     !subst("DWORDX3", "DWORD",
 280     !subst("DWORDX4", "DWORD", Op)));
 281 }
 282
 283 class MUBUF_Pseudo <string opName, dag outs, dag ins,
 284                     string asmOps, list<dag> pattern=[]> :
 285   InstSI<outs, ins, "", pattern>,
 286   SIMCInstr<opName, SIEncodingFamily.NONE> {
 287
 288   let isPseudo = 1;
 289   let isCodeGenOnly = 1;
 290   let Size = 8;
 291   let UseNamedOperandTable = 1;
 292
 293   string Mnemonic = opName;
 294   string AsmOperands = asmOps;
 295
 296   Instruction Opcode = !cast<Instruction>(NAME);
 297   Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret);
 298
 299   let VM_CNT = 1;
 300   let EXP_CNT = 1;
 301   let MUBUF = 1;
 302   let Uses = [EXEC];
 303   let hasSideEffects = 0;
 304   let SchedRW = [WriteVMEM];
 305
 306   let AsmMatchConverter = "cvtMubuf";
 307
 308   bits<1> offen       = 0;
 309   bits<1> idxen       = 0;
 310   bits<1> addr64      = 0;
 311   bits<1> lds         = 0;
 312   bits<1> has_vdata   = 1;
 313   bits<1> has_vaddr   = 1;
 314   bits<1> has_glc     = 1;
 315   bits<1> has_dlc     = 1;
 316   bits<1> glc_value   = 0; // the value for glc if no such operand
 317   bits<1> dlc_value   = 0; // the value for dlc if no such operand
 318   bits<1> has_srsrc   = 1;
 319   bits<1> has_soffset = 1;
 320   bits<1> has_offset  = 1;
 321   bits<1> has_slc     = 1;
 322   bits<1> has_tfe     = 1;
 323   bits<4> dwords      = 0;
 324 }
 325
 326 class MUBUF_Real <MUBUF_Pseudo ps> :
 327   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 328
 329   let isPseudo = 0;
 330   let isCodeGenOnly = 0;
 331
 332   // copy relevant pseudo op flags
 333   let SubtargetPredicate = ps.SubtargetPredicate;
 334   let AsmMatchConverter  = ps.AsmMatchConverter;
 335   let Constraints        = ps.Constraints;
 336   let DisableEncoding    = ps.DisableEncoding;
 337   let TSFlags            = ps.TSFlags;
 338
 339   bits<12> offset;
 340   bits<1>  glc;
 341   bits<1>  dlc;
 342   bits<8>  vaddr;
 343   bits<8>  vdata;
 344   bits<7>  srsrc;
 345   bits<1>  slc;
 346   bits<1>  tfe;
 347   bits<8>  soffset;
 348 }
 349
 350
 351 // For cache invalidation instructions.
 352 class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> :
 353   MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {
 354
 355   let AsmMatchConverter = "";
 356
 357   let hasSideEffects = 1;
 358   let mayStore = 1;
 359
 360   // Set everything to 0.
 361   let offen       = 0;
 362   let idxen       = 0;
 363   let addr64      = 0;
 364   let has_vdata   = 0;
 365   let has_vaddr   = 0;
 366   let has_glc     = 0;
 367   let has_dlc     = 0;
 368   let glc_value   = 0;
 369   let dlc_value   = 0;
 370   let has_srsrc   = 0;
 371   let has_soffset = 0;
 372   let has_offset  = 0;
 373   let has_slc     = 0;
 374   let has_tfe     = 0;
 375 }
 376
 377 class getMUBUFInsDA<list<RegisterClass> vdataList,
 378                     list<RegisterClass> vaddrList=[],
 379                     bit isLds = 0> {
 380   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 381   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 382   dag InsNoData = !if(!empty(vaddrList),
 383     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 384          offset:$offset, GLC:$glc, SLC:$slc),
 385     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 386          offset:$offset, GLC:$glc, SLC:$slc)
 387   );
 388   dag InsData = !if(!empty(vaddrList),
 389     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 390          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc),
 391     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 392          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc)
 393   );
 394   dag ret = !con(
 395               !if(!empty(vdataList), InsNoData, InsData),
 396               !if(isLds, (ins DLC:$dlc), (ins TFE:$tfe, DLC:$dlc))
 397              );
 398 }
 399
 400 class getMUBUFDwords<RegisterClass regClass> {
 401   string regClassAsInt = !cast<string>(regClass);
 402   int ret =
 403     !if(!eq(regClassAsInt, !cast<string>(VGPR_32)), 1,
 404     !if(!eq(regClassAsInt, !cast<string>(VReg_64)), 2,
 405     !if(!eq(regClassAsInt, !cast<string>(VReg_96)), 3,
 406     !if(!eq(regClassAsInt, !cast<string>(VReg_128)), 4,
 407     0))));
 408 }
 409
 410 class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> {
 411   dag ret =
 412     !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret,
 413     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 414     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 415     !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 416     !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 417     (ins))))));
 418 }
 419
 420 class getMUBUFAsmOps<int addrKind> {
 421   string Pfx =
 422     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",
 423     !if(!eq(addrKind, BUFAddrKind.OffEn),  "$vaddr, $srsrc, $soffset offen",
 424     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "$vaddr, $srsrc, $soffset idxen",
 425     !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",
 426     !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",
 427     "")))));
 428   string ret = Pfx # "$offset";
 429 }
 430
 431 class MUBUF_SetupAddr<int addrKind> {
 432   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 433                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 434
 435   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 436                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 437
 438   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 439
 440   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 441 }
 442
 443 class MUBUF_Load_Pseudo <string opName,
 444                          int addrKind,
 445                          RegisterClass vdataClass,
 446                          bit HasTiedDest = 0,
 447                          bit isLds = 0,
 448                          list<dag> pattern=[],
 449                          // Workaround bug bz30254
 450                          int addrKindCopy = addrKind>
 451   : MUBUF_Pseudo<opName,
 452                  (outs vdataClass:$vdata),
 453                  !con(getMUBUFIns<addrKindCopy, [], isLds>.ret,
 454                       !if(HasTiedDest, (ins vdataClass:$vdata_in), (ins))),
 455                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" #
 456                    !if(isLds, " lds", "$tfe") # "$dlc",
 457                  pattern>,
 458     MUBUF_SetupAddr<addrKindCopy> {
 459   let PseudoInstr = opName # !if(isLds, "_lds", "") #
 460                     "_" # getAddrName<addrKindCopy>.ret;
 461   let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf");
 462
 463   let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", "");
 464   let mayLoad = 1;
 465   let mayStore = 0;
 466   let maybeAtomic = 1;
 467   let Uses = !if(isLds, [EXEC, M0], [EXEC]);
 468   let has_tfe = !if(isLds, 0, 1);
 469   let lds = isLds;
 470   let dwords = getMUBUFDwords<vdataClass>.ret;
 471 }
 472
 473 class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat <
 474   (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
 475   (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))
 476 >;
 477
 478 class MUBUF_Addr64_Load_Pat <Instruction inst,
 479                             ValueType load_vt = i32,
 480                             SDPatternOperator ld = null_frag> : Pat <
 481   (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
 482   (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))
 483 >;
 484
 485 multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> {
 486   def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>;
 487   def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>;
 488 }
 489
 490
 491 // FIXME: tfe can't be an operand because it requires a separate
 492 // opcode because it needs an N+1 register class dest register.
 493 multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 494                               ValueType load_vt = i32,
 495                               SDPatternOperator ld = null_frag,
 496                               bit TiedDest = 0,
 497                               bit isLds = 0> {
 498
 499   def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, TiedDest, isLds>,
 500     MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>;
 501
 502   def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, TiedDest, isLds>,
 503     MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>;
 504
 505   def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>;
 506   def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>;
 507   def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>;
 508
 509   let DisableWQM = 1 in {
 510     def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, TiedDest, isLds>;
 511     def _OFFEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>;
 512     def _IDXEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>;
 513     def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>;
 514   }
 515 }
 516
 517 multiclass MUBUF_Pseudo_Loads_Lds<string opName, RegisterClass vdataClass,
 518                                   ValueType load_vt = i32,
 519                                   SDPatternOperator ld_nolds = null_frag,
 520                                   SDPatternOperator ld_lds = null_frag> {
 521   defm NAME : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_nolds>;
 522   defm _LDS : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_lds, 0, 1>;
 523 }
 524
 525 class MUBUF_Store_Pseudo <string opName,
 526                           int addrKind,
 527                           RegisterClass vdataClass,
 528                           list<dag> pattern=[],
 529                           // Workaround bug bz30254
 530                           int addrKindCopy = addrKind,
 531                           RegisterClass vdataClassCopy = vdataClass>
 532   : MUBUF_Pseudo<opName,
 533                  (outs),
 534                  getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 535                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc",
 536                  pattern>,
 537     MUBUF_SetupAddr<addrKindCopy> {
 538   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 539   let mayLoad = 0;
 540   let mayStore = 1;
 541   let maybeAtomic = 1;
 542   let dwords = getMUBUFDwords<vdataClass>.ret;
 543 }
 544
 545 multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 546                                ValueType store_vt = i32,
 547                                SDPatternOperator st = null_frag> {
 548
 549   def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 550     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 551                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))]>,
 552     MUBUFAddr64Table<0, NAME>;
 553
 554   def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 555     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 556                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))]>,
 557     MUBUFAddr64Table<1, NAME>;
 558
 559   def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 560   def _IDXEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 561   def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 562
 563   let DisableWQM = 1 in {
 564     def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 565     def _OFFEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 566     def _IDXEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 567     def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 568   }
 569 }
 570
 571 class MUBUF_Pseudo_Store_Lds<string opName>
 572   : MUBUF_Pseudo<opName,
 573                  (outs),
 574                  (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc),
 575                  " $srsrc, $soffset$offset lds$glc$slc"> {
 576   let mayLoad = 0;
 577   let mayStore = 1;
 578   let maybeAtomic = 1;
 579
 580   let has_vdata = 0;
 581   let has_vaddr = 0;
 582   let has_tfe = 0;
 583   let lds = 1;
 584
 585   let Uses = [EXEC, M0];
 586   let AsmMatchConverter = "cvtMubufLds";
 587 }
 588
 589 class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
 590                           list<RegisterClass> vaddrList=[]> {
 591   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 592   dag ret = !if(vdata_in,
 593     !if(!empty(vaddrList),
 594       (ins vdataClass:$vdata_in,
 595            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc),
 596       (ins vdataClass:$vdata_in, vaddrClass:$vaddr,
 597            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc)
 598     ),
 599     !if(!empty(vaddrList),
 600       (ins vdataClass:$vdata,
 601            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc),
 602       (ins vdataClass:$vdata, vaddrClass:$vaddr,
 603            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc)
 604   ));
 605 }
 606
 607 class getMUBUFAtomicIns<int addrKind,
 608                         RegisterClass vdataClass,
 609                         bit vdata_in,
 610                         // Workaround bug bz30254
 611                         RegisterClass vdataClassCopy=vdataClass> {
 612   dag ret =
 613     !if(!eq(addrKind, BUFAddrKind.Offset),
 614             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret,
 615     !if(!eq(addrKind, BUFAddrKind.OffEn),
 616             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 617     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 618             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 619     !if(!eq(addrKind, BUFAddrKind.BothEn),
 620             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 621     !if(!eq(addrKind, BUFAddrKind.Addr64),
 622             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 623     (ins))))));
 624 }
 625
 626 class MUBUF_Atomic_Pseudo<string opName,
 627                           int addrKind,
 628                           dag outs,
 629                           dag ins,
 630                           string asmOps,
 631                           list<dag> pattern=[],
 632                           // Workaround bug bz30254
 633                           int addrKindCopy = addrKind>
 634   : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,
 635     MUBUF_SetupAddr<addrKindCopy> {
 636   let mayStore = 1;
 637   let mayLoad = 1;
 638   let hasPostISelHook = 1;
 639   let hasSideEffects = 1;
 640   let DisableWQM = 1;
 641   let has_glc = 0;
 642   let has_dlc = 0;
 643   let has_tfe = 0;
 644   let maybeAtomic = 1;
 645 }
 646
 647 class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
 648                                RegisterClass vdataClass,
 649                                list<dag> pattern=[],
 650                                // Workaround bug bz30254
 651                                int addrKindCopy = addrKind,
 652                                RegisterClass vdataClassCopy = vdataClass>
 653   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 654                         (outs),
 655                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret,
 656                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc",
 657                         pattern>,
 658     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> {
 659   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 660   let glc_value = 0;
 661   let dlc_value = 0;
 662   let AsmMatchConverter = "cvtMubufAtomic";
 663 }
 664
 665 class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
 666                              RegisterClass vdataClass,
 667                              list<dag> pattern=[],
 668                              // Workaround bug bz30254
 669                              int addrKindCopy = addrKind,
 670                              RegisterClass vdataClassCopy = vdataClass>
 671   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 672                         (outs vdataClassCopy:$vdata),
 673                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret,
 674                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc",
 675                         pattern>,
 676     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> {
 677   let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret;
 678   let glc_value = 1;
 679   let dlc_value = 0;
 680   let Constraints = "$vdata = $vdata_in";
 681   let DisableEncoding = "$vdata_in";
 682   let AsmMatchConverter = "cvtMubufAtomicReturn";
 683 }
 684
 685 multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName,
 686                                         RegisterClass vdataClass,
 687                                         ValueType vdataType,
 688                                         SDPatternOperator atomic,
 689                                         bit isFP = getIsFP<vdataType>.ret> {
 690   let FPAtomic = isFP in
 691   def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>,
 692                 MUBUFAddr64Table <0, NAME>;
 693
 694   let FPAtomic = isFP in
 695   def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>,
 696                 MUBUFAddr64Table <1, NAME>;
 697
 698   let FPAtomic = isFP in
 699   def _OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 700
 701   let FPAtomic = isFP in
 702
 703   def _IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 704
 705   let FPAtomic = isFP in
 706   def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 707 }
 708
 709 multiclass MUBUF_Pseudo_Atomics_RTN <string opName,
 710                                      RegisterClass vdataClass,
 711                                      ValueType vdataType,
 712                                      SDPatternOperator atomic,
 713                                      bit isFP = getIsFP<vdataType>.ret> {
 714   let FPAtomic = isFP in
 715   def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 716     [(set vdataType:$vdata,
 717      (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc),
 718              vdataType:$vdata_in))]>,
 719     MUBUFAddr64Table <0, NAME # "_RTN">;
 720
 721   let FPAtomic = isFP in
 722   def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 723     [(set vdataType:$vdata,
 724      (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc),
 725              vdataType:$vdata_in))]>,
 726     MUBUFAddr64Table <1, NAME # "_RTN">;
 727
 728   let FPAtomic = isFP in
 729   def _OFFEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 730
 731   let FPAtomic = isFP in
 732   def _IDXEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 733
 734   let FPAtomic = isFP in
 735   def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 736 }
 737
 738 multiclass MUBUF_Pseudo_Atomics <string opName,
 739                                  RegisterClass vdataClass,
 740                                  ValueType vdataType,
 741                                  SDPatternOperator atomic> :
 742   MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>,
 743   MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>;
 744
 745
 746 //===----------------------------------------------------------------------===//
 747 // MUBUF Instructions
 748 //===----------------------------------------------------------------------===//
 749
 750 defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds <
 751   "buffer_load_format_x", VGPR_32
 752 >;
 753 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
 754   "buffer_load_format_xy", VReg_64
 755 >;
 756 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <
 757   "buffer_load_format_xyz", VReg_96
 758 >;
 759 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <
 760   "buffer_load_format_xyzw", VReg_128
 761 >;
 762 defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <
 763   "buffer_store_format_x", VGPR_32
 764 >;
 765 defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <
 766   "buffer_store_format_xy", VReg_64
 767 >;
 768 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <
 769   "buffer_store_format_xyz", VReg_96
 770 >;
 771 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <
 772   "buffer_store_format_xyzw", VReg_128
 773 >;
 774
 775 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
 776   defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads <
 777     "buffer_load_format_d16_x", VGPR_32
 778   >;
 779   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads <
 780     "buffer_load_format_d16_xy", VReg_64
 781   >;
 782   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads <
 783     "buffer_load_format_d16_xyz", VReg_96
 784   >;
 785   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads <
 786    "buffer_load_format_d16_xyzw", VReg_128
 787   >;
 788   defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores <
 789     "buffer_store_format_d16_x", VGPR_32
 790   >;
 791   defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores <
 792     "buffer_store_format_d16_xy", VReg_64
 793   >;
 794   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores <
 795     "buffer_store_format_d16_xyz", VReg_96
 796   >;
 797   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores <
 798     "buffer_store_format_d16_xyzw", VReg_128
 799   >;
 800 } // End HasUnpackedD16VMem.
 801
 802 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
 803   defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads <
 804     "buffer_load_format_d16_x", VGPR_32
 805   >;
 806   defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads <
 807     "buffer_load_format_d16_xy", VGPR_32
 808   >;
 809   defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads <
 810     "buffer_load_format_d16_xyz", VReg_64
 811   >;
 812   defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads <
 813     "buffer_load_format_d16_xyzw", VReg_64
 814   >;
 815   defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores <
 816     "buffer_store_format_d16_x", VGPR_32
 817   >;
 818   defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores <
 819     "buffer_store_format_d16_xy", VGPR_32
 820   >;
 821   defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores <
 822     "buffer_store_format_d16_xyz", VReg_64
 823   >;
 824   defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores <
 825     "buffer_store_format_d16_xyzw", VReg_64
 826   >;
 827 } // End HasPackedD16VMem.
 828
 829 defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds <
 830   "buffer_load_ubyte", VGPR_32, i32
 831 >;
 832 defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds <
 833   "buffer_load_sbyte", VGPR_32, i32
 834 >;
 835 defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds <
 836   "buffer_load_ushort", VGPR_32, i32
 837 >;
 838 defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds <
 839   "buffer_load_sshort", VGPR_32, i32
 840 >;
 841 defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds <
 842   "buffer_load_dword", VGPR_32, i32
 843 >;
 844 defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
 845   "buffer_load_dwordx2", VReg_64, v2i32
 846 >;
 847 defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
 848   "buffer_load_dwordx3", VReg_96, v3i32
 849 >;
 850 defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
 851   "buffer_load_dwordx4", VReg_128, v4i32
 852 >;
 853
 854 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>;
 855 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>;
 856 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>;
 857 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>;
 858 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>;
 859 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>;
 860 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>;
 861 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>;
 862 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>;
 863 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>;
 864
 865 // This is not described in AMD documentation,
 866 // but 'lds' versions of these opcodes are available
 867 // in at least GFX8+ chips. See Bug 37653.
 868 let SubtargetPredicate = isGFX8GFX9 in {
 869 defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads <
 870   "buffer_load_dwordx2", VReg_64, v2i32, null_frag, 0, 1
 871 >;
 872 defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads <
 873   "buffer_load_dwordx3", VReg_96, untyped, null_frag, 0, 1
 874 >;
 875 defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads <
 876   "buffer_load_dwordx4", VReg_128, v4i32, null_frag, 0, 1
 877 >;
 878 }
 879
 880 defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
 881   "buffer_store_byte", VGPR_32, i32, truncstorei8_global
 882 >;
 883 defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
 884   "buffer_store_short", VGPR_32, i32, truncstorei16_global
 885 >;
 886 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
 887   "buffer_store_dword", VGPR_32, i32, store_global
 888 >;
 889 defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
 890   "buffer_store_dwordx2", VReg_64, v2i32, store_global
 891 >;
 892 defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
 893   "buffer_store_dwordx3", VReg_96, v3i32, store_global
 894 >;
 895 defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
 896   "buffer_store_dwordx4", VReg_128, v4i32, store_global
 897 >;
 898 defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
 899   "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global
 900 >;
 901 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <
 902   "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag
 903 >;
 904 defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <
 905   "buffer_atomic_add", VGPR_32, i32, atomic_add_global
 906 >;
 907 defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <
 908   "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global
 909 >;
 910 defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <
 911   "buffer_atomic_smin", VGPR_32, i32, atomic_min_global
 912 >;
 913 defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <
 914   "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global
 915 >;
 916 defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <
 917   "buffer_atomic_smax", VGPR_32, i32, atomic_max_global
 918 >;
 919 defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <
 920   "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global
 921 >;
 922 defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <
 923   "buffer_atomic_and", VGPR_32, i32, atomic_and_global
 924 >;
 925 defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <
 926   "buffer_atomic_or", VGPR_32, i32, atomic_or_global
 927 >;
 928 defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <
 929   "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global
 930 >;
 931 defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <
 932   "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global
 933 >;
 934 defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <
 935   "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global
 936 >;
 937 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <
 938   "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global
 939 >;
 940 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <
 941   "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag
 942 >;
 943 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <
 944   "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global
 945 >;
 946 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <
 947   "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global
 948 >;
 949 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <
 950   "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global
 951 >;
 952 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <
 953   "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global
 954 >;
 955 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <
 956   "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global
 957 >;
 958 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <
 959   "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global
 960 >;
 961 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <
 962   "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global
 963 >;
 964 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <
 965   "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global
 966 >;
 967 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <
 968   "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global
 969 >;
 970 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <
 971   "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global
 972 >;
 973 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
 974   "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global
 975 >;
 976
 977 let SubtargetPredicate = isGFX8GFX9 in {
 978 def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">;
 979 }
 980
 981 let SubtargetPredicate = isGFX6 in { // isn't on CI & VI
 982 /*
 983 defm BUFFER_ATOMIC_RSUB        : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
 984 defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">;
 985 defm BUFFER_ATOMIC_FMIN        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">;
 986 defm BUFFER_ATOMIC_FMAX        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">;
 987 defm BUFFER_ATOMIC_RSUB_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
 988 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">;
 989 defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">;
 990 defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">;
 991 */
 992
 993 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
 994                                           int_amdgcn_buffer_wbinvl1_sc>;
 995 }
 996
 997 let SubtargetPredicate = HasD16LoadStore in {
 998
 999 defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads <
1000   "buffer_load_ubyte_d16", VGPR_32, i32, null_frag, 1
1001 >;
1002
1003 defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads <
1004   "buffer_load_ubyte_d16_hi", VGPR_32, i32, null_frag, 1
1005 >;
1006
1007 defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads <
1008   "buffer_load_sbyte_d16", VGPR_32, i32, null_frag, 1
1009 >;
1010
1011 defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads <
1012   "buffer_load_sbyte_d16_hi", VGPR_32, i32, null_frag, 1
1013 >;
1014
1015 defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads <
1016   "buffer_load_short_d16", VGPR_32, i32, null_frag, 1
1017 >;
1018
1019 defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads <
1020   "buffer_load_short_d16_hi", VGPR_32, i32, null_frag, 1
1021 >;
1022
1023 defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores <
1024   "buffer_store_byte_d16_hi", VGPR_32, i32
1025 >;
1026
1027 defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores <
1028   "buffer_store_short_d16_hi", VGPR_32, i32
1029 >;
1030
1031 defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads <
1032   "buffer_load_format_d16_hi_x", VGPR_32
1033 >;
1034 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores <
1035   "buffer_store_format_d16_hi_x", VGPR_32
1036 >;
1037
1038 } // End HasD16LoadStore
1039
1040 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
1041                                        int_amdgcn_buffer_wbinvl1>;
1042
1043 let SubtargetPredicate = HasAtomicFaddInsts in {
1044
1045 defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN <
1046   "buffer_atomic_add_f32", VGPR_32, f32, atomic_add_global
1047 >;
1048 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN <
1049   "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_add_global
1050 >;
1051
1052 } // End SubtargetPredicate = HasAtomicFaddInsts
1053
1054 //===----------------------------------------------------------------------===//
1055 // MTBUF Instructions
1056 //===----------------------------------------------------------------------===//
1057
1058 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32>;
1059 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xy",    VReg_64>;
1060 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyz",   VReg_96>;
1061 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyzw",  VReg_128>;
1062 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_x",    VGPR_32>;
1063 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy",   VReg_64>;
1064 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz",  VReg_96>;
1065 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>;
1066
1067 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
1068   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32>;
1069   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VReg_64>;
1070   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_96>;
1071   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_128>;
1072   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32>;
1073   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VReg_64>;
1074   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_96>;
1075   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128>;
1076 } // End HasUnpackedD16VMem.
1077
1078 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
1079   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32>;
1080   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VGPR_32>;
1081   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_64>;
1082   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_64>;
1083   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32>;
1084   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VGPR_32>;
1085   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_64>;
1086   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64>;
1087 } // End HasPackedD16VMem.
1088
1089 let SubtargetPredicate = isGFX7Plus in {
1090
1091 //===----------------------------------------------------------------------===//
1092 // Instruction definitions for CI and newer.
1093 //===----------------------------------------------------------------------===//
1094
1095 def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
1096                                            int_amdgcn_buffer_wbinvl1_vol>;
1097
1098 } // End let SubtargetPredicate = isGFX7Plus
1099
1100 let SubtargetPredicate = isGFX10Plus in {
1101   def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">;
1102   def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">;
1103 } // End SubtargetPredicate = isGFX10Plus
1104
1105 //===----------------------------------------------------------------------===//
1106 // MUBUF Patterns
1107 //===----------------------------------------------------------------------===//
1108
1109 def extract_glc : SDNodeXForm<imm, [{
1110   return CurDAG->getTargetConstant(N->getZExtValue() & 1, SDLoc(N), MVT::i8);
1111 }]>;
1112
1113 def extract_slc : SDNodeXForm<imm, [{
1114   return CurDAG->getTargetConstant((N->getZExtValue() >> 1) & 1, SDLoc(N), MVT::i8);
1115 }]>;
1116
1117 def extract_dlc : SDNodeXForm<imm, [{
1118   return CurDAG->getTargetConstant((N->getZExtValue() >> 2) & 1, SDLoc(N), MVT::i8);
1119 }]>;
1120
1121 //===----------------------------------------------------------------------===//
1122 // buffer_load/store_format patterns
1123 //===----------------------------------------------------------------------===//
1124
1125 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1126                                   string opcode> {
1127   def : GCNPat<
1128     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1129               imm:$cachepolicy, 0)),
1130     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
1131       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1132   >;
1133
1134   def : GCNPat<
1135     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1136               imm:$cachepolicy, 0)),
1137     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
1138       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1139   >;
1140
1141   def : GCNPat<
1142     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1143               imm:$cachepolicy, imm)),
1144     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
1145       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1146   >;
1147
1148   def : GCNPat<
1149     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset,
1150               imm:$cachepolicy, imm)),
1151     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1152       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1153       $rsrc, $soffset, (as_i16imm $offset),
1154       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1155   >;
1156 }
1157
1158 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
1159 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">;
1160 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
1161 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">;
1162 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">;
1163 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">;
1164 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;
1165 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">;
1166
1167 let SubtargetPredicate = HasUnpackedD16VMem in {
1168   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1169   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1170   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1171   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1172 } // End HasUnpackedD16VMem.
1173
1174 let SubtargetPredicate = HasPackedD16VMem in {
1175   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">;
1176   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">;
1177   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">;
1178   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">;
1179   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1180   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1181 } // End HasPackedD16VMem.
1182
1183 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">;
1184 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">;
1185 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">;
1186 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">;
1187 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">;
1188 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">;
1189 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">;
1190 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">;
1191 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">;
1192 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">;
1193 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">;
1194 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort,  i32, "BUFFER_LOAD_USHORT">;
1195
1196 multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1197                                    string opcode> {
1198   def : GCNPat<
1199     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1200               imm:$cachepolicy, 0),
1201     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset),
1202       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1203   >;
1204
1205   def : GCNPat<
1206     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1207               imm:$cachepolicy, 0),
1208     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
1209       (as_i16imm $offset), (extract_glc $cachepolicy),
1210       (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1211   >;
1212
1213   def : GCNPat<
1214     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1215               imm:$cachepolicy, imm),
1216     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
1217       (as_i16imm $offset), (extract_glc $cachepolicy),
1218       (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1219   >;
1220
1221   def : GCNPat<
1222     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset,
1223               imm:$cachepolicy, imm),
1224     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
1225       $vdata,
1226       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1227       $rsrc, $soffset, (as_i16imm $offset), (extract_glc $cachepolicy),
1228       (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1229   >;
1230 }
1231
1232 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
1233 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">;
1234 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
1235 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">;
1236 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">;
1237 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">;
1238 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
1239 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">;
1240
1241 let SubtargetPredicate = HasUnpackedD16VMem in {
1242   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1243   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1244   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">;
1245   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1246 } // End HasUnpackedD16VMem.
1247
1248 let SubtargetPredicate = HasPackedD16VMem in {
1249   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">;
1250   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">;
1251   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">;
1252   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">;
1253   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1254   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1255 } // End HasPackedD16VMem.
1256
1257 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">;
1258 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">;
1259 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">;
1260 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">;
1261 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">;
1262 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">;
1263 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">;
1264 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">;
1265 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">;
1266 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">;
1267
1268 //===----------------------------------------------------------------------===//
1269 // buffer_atomic patterns
1270 //===----------------------------------------------------------------------===//
1271
1272 multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt,
1273                                 string opcode> {
1274   def : GCNPat<
1275     (vt (name vt:$vdata_in, v4i32:$rsrc, 0,
1276           0, i32:$soffset, imm:$offset,
1277           imm:$cachepolicy, 0)),
1278     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset,
1279                                         (as_i16imm $offset), (extract_slc $cachepolicy))
1280   >;
1281
1282   def : GCNPat<
1283     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1284           0, i32:$soffset, imm:$offset,
1285           imm:$cachepolicy, imm)),
1286     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset,
1287                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1288   >;
1289
1290   def : GCNPat<
1291     (vt (name vt:$vdata_in, v4i32:$rsrc, 0,
1292           i32:$voffset, i32:$soffset, imm:$offset,
1293           imm:$cachepolicy, 0)),
1294     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset,
1295                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1296   >;
1297
1298   def : GCNPat<
1299     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1300           i32:$voffset, i32:$soffset, imm:$offset,
1301           imm:$cachepolicy, imm)),
1302     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN)
1303       $vdata_in,
1304       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1305       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy))
1306   >;
1307 }
1308
1309 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">;
1310 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">;
1311 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">;
1312 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">;
1313 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">;
1314 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">;
1315 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">;
1316 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">;
1317 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">;
1318 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">;
1319 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">;
1320 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64,  "BUFFER_ATOMIC_ADD_X2">;
1321 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">;
1322 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">;
1323 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">;
1324 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">;
1325 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">;
1326 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">;
1327 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">;
1328 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">;
1329
1330 multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt,
1331                                        string opcode> {
1332   def : GCNPat<
1333     (name vt:$vdata_in, v4i32:$rsrc, 0,
1334           0, i32:$soffset, imm:$offset,
1335           imm:$cachepolicy, 0),
1336     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $vdata_in, $rsrc, $soffset,
1337                                         (as_i16imm $offset), (extract_slc $cachepolicy))
1338   >;
1339
1340   def : GCNPat<
1341     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1342           0, i32:$soffset, imm:$offset,
1343           imm:$cachepolicy, imm),
1344     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vdata_in, $vindex, $rsrc, $soffset,
1345                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1346   >;
1347
1348   def : GCNPat<
1349     (name vt:$vdata_in, v4i32:$rsrc, 0,
1350           i32:$voffset, i32:$soffset, imm:$offset,
1351           imm:$cachepolicy, 0),
1352     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $vdata_in, $voffset, $rsrc, $soffset,
1353                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1354   >;
1355
1356   def : GCNPat<
1357     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1358           i32:$voffset, i32:$soffset, imm:$offset,
1359           imm:$cachepolicy, imm),
1360     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1361       $vdata_in,
1362       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1363       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy))
1364   >;
1365 }
1366
1367 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">;
1368 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_pk_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">;
1369
1370 def : GCNPat<
1371   (SIbuffer_atomic_cmpswap
1372       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1373       0, i32:$soffset, imm:$offset,
1374       imm:$cachepolicy, 0),
1375   (EXTRACT_SUBREG
1376     (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN
1377       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1378       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1379     sub0)
1380 >;
1381
1382 def : GCNPat<
1383   (SIbuffer_atomic_cmpswap
1384       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1385       0, i32:$soffset, imm:$offset,
1386       imm:$cachepolicy, imm),
1387   (EXTRACT_SUBREG
1388     (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN
1389       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1390       $vindex, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1391     sub0)
1392 >;
1393
1394 def : GCNPat<
1395   (SIbuffer_atomic_cmpswap
1396       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1397       i32:$voffset, i32:$soffset, imm:$offset,
1398       imm:$cachepolicy, 0),
1399   (EXTRACT_SUBREG
1400     (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN
1401       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1402       $voffset, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1403     sub0)
1404 >;
1405
1406 def : GCNPat<
1407   (SIbuffer_atomic_cmpswap
1408       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1409       i32:$voffset, i32:$soffset, imm:$offset,
1410       imm:$cachepolicy, imm),
1411   (EXTRACT_SUBREG
1412     (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN
1413       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1414       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1415       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1416     sub0)
1417 >;
1418
1419 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
1420                               PatFrag constant_ld> : GCNPat <
1421      (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1422                                    i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
1423      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc)
1424   >;
1425
1426 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1427                                      ValueType vt, PatFrag atomic_ld> {
1428   def : GCNPat <
1429      (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1430                                    i16:$offset, i1:$slc))),
1431      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0)
1432   >;
1433
1434   def : GCNPat <
1435     (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))),
1436     (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0)
1437   >;
1438 }
1439
1440 let SubtargetPredicate = isGFX6GFX7 in {
1441 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
1442 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>;
1443 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>;
1444 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
1445 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>;
1446 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>;
1447
1448 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>;
1449 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>;
1450 } // End SubtargetPredicate = isGFX6GFX7
1451
1452 multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1453                                PatFrag ld> {
1454
1455   def : GCNPat <
1456     (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1457                           i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
1458     (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc)
1459   >;
1460 }
1461
1462 let OtherPredicates = [Has16BitInsts] in {
1463
1464 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>;
1465 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>;
1466 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>;
1467 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>;
1468 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>;
1469 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>;
1470
1471 defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>;
1472
1473 } // End OtherPredicates = [Has16BitInsts]
1474
1475 multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen,
1476                                 MUBUF_Pseudo InstrOffset,
1477                                 ValueType vt, PatFrag ld> {
1478   def : GCNPat <
1479     (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1480                                i32:$soffset, u16imm:$offset))),
1481     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0)
1482   >;
1483
1484   def : GCNPat <
1485     (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))),
1486     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0)
1487   >;
1488 }
1489
1490 // XXX - Is it possible to have a complex pattern in a PatFrag?
1491 multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen,
1492                                 MUBUF_Pseudo InstrOffset,
1493                                 ValueType vt, PatFrag ld_frag> {
1494   def : GCNPat <
1495     (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in),
1496     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, $in)
1497   >;
1498
1499   def : GCNPat <
1500     (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in),
1501     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, $in)
1502   >;
1503 }
1504
1505 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>;
1506 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>;
1507 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>;
1508 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>;
1509 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>;
1510 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>;
1511 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>;
1512 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>;
1513 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>;
1514 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>;
1515 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>;
1516 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>;
1517 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>;
1518 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>;
1519
1520 let OtherPredicates = [D16PreservesUnusedBits] in {
1521 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>;
1522 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>;
1523 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>;
1524 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>;
1525 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>;
1526 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>;
1527
1528 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>;
1529 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>;
1530 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>;
1531 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>;
1532 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>;
1533 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>;
1534 }
1535
1536 multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1537                                       ValueType vt, PatFrag atomic_st> {
1538   // Store follows atomic op convention so address is forst
1539   def : GCNPat <
1540      (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1541                                    i16:$offset, i1:$slc), vt:$val),
1542      (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0)
1543   >;
1544
1545   def : GCNPat <
1546     (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
1547     (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0)
1548   >;
1549 }
1550 let SubtargetPredicate = isGFX6GFX7 in {
1551 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>;
1552 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>;
1553 } // End Predicates = isGFX6GFX7
1554
1555
1556 multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1557                                PatFrag st> {
1558
1559   def : GCNPat <
1560     (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1561                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)),
1562     (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc)
1563   >;
1564 }
1565
1566 defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>;
1567 defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>;
1568
1569 multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen,
1570                                  MUBUF_Pseudo InstrOffset,
1571                                  ValueType vt, PatFrag st,
1572                                  RegisterClass rc = VGPR_32> {
1573   def : GCNPat <
1574     (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1575                                       i32:$soffset, u16imm:$offset)),
1576     (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0)
1577   >;
1578
1579   def : GCNPat <
1580     (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,
1581                                        u16imm:$offset)),
1582     (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0)
1583   >;
1584 }
1585
1586 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>;
1587 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>;
1588 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>;
1589 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>;
1590 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>;
1591 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>;
1592 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>;
1593 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>;
1594
1595
1596 let OtherPredicates = [D16PreservesUnusedBits] in {
1597  // Hiding the extract high pattern in the PatFrag seems to not
1598  // automatically increase the complexity.
1599 let AddedComplexity = 1 in {
1600 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>;
1601 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>;
1602 }
1603 }
1604
1605 //===----------------------------------------------------------------------===//
1606 // MTBUF Patterns
1607 //===----------------------------------------------------------------------===//
1608
1609 //===----------------------------------------------------------------------===//
1610 // tbuffer_load/store_format patterns
1611 //===----------------------------------------------------------------------===//
1612
1613 multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1614                                   string opcode> {
1615   def : GCNPat<
1616     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1617               imm:$format, imm:$cachepolicy, 0)),
1618     (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
1619       (as_i8imm $format),
1620       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1621   >;
1622
1623   def : GCNPat<
1624     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1625               imm:$format, imm:$cachepolicy, imm)),
1626     (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
1627       (as_i8imm $format),
1628       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1629   >;
1630
1631   def : GCNPat<
1632     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1633               imm:$format, imm:$cachepolicy, 0)),
1634     (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
1635       (as_i8imm $format),
1636       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1637   >;
1638
1639   def : GCNPat<
1640     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset,
1641               imm:$format, imm:$cachepolicy, imm)),
1642     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN)
1643       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1644       $rsrc, $soffset, (as_i16imm $offset),
1645       (as_i8imm $format),
1646       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1647   >;
1648 }
1649
1650 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32,   "TBUFFER_LOAD_FORMAT_X">;
1651 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;
1652 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">;
1653 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">;
1654 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32,   "TBUFFER_LOAD_FORMAT_X">;
1655 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;
1656 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">;
1657 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;
1658
1659 let SubtargetPredicate = HasUnpackedD16VMem in {
1660   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
1661   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1662   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1663 } // End HasUnpackedD16VMem.
1664
1665 let SubtargetPredicate = HasPackedD16VMem in {
1666   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X">;
1667   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">;
1668   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
1669 } // End HasPackedD16VMem.
1670
1671 multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1672                                    string opcode> {
1673   def : GCNPat<
1674     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1675           imm:$format, imm:$cachepolicy, 0),
1676     (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset,
1677       (as_i16imm $offset), (as_i8imm $format),
1678       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1679   >;
1680
1681   def : GCNPat<
1682     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1683           imm:$format, imm:$cachepolicy, imm),
1684     (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
1685       (as_i16imm $offset), (as_i8imm $format),
1686       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1687   >;
1688
1689   def : GCNPat<
1690     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1691           imm:$format, imm:$cachepolicy, 0),
1692     (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
1693       (as_i16imm $offset), (as_i8imm $format),
1694       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1695   >;
1696
1697   def : GCNPat<
1698     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset,
1699           imm:$offset, imm:$format, imm:$cachepolicy, imm),
1700     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact)
1701       $vdata,
1702       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1703       $rsrc, $soffset, (as_i16imm $offset), (as_i8imm $format),
1704       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1705   >;
1706 }
1707
1708 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32,   "TBUFFER_STORE_FORMAT_X">;
1709 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;
1710 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">;
1711 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">;
1712 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32,   "TBUFFER_STORE_FORMAT_X">;
1713 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;
1714 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">;
1715 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;
1716
1717 let SubtargetPredicate = HasUnpackedD16VMem in {
1718   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
1719   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">;
1720   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1721 } // End HasUnpackedD16VMem.
1722
1723 let SubtargetPredicate = HasPackedD16VMem in {
1724   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X">;
1725   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">;
1726   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
1727 } // End HasPackedD16VMem.
1728
1729 //===----------------------------------------------------------------------===//
1730 // Target-specific instruction encodings.
1731 //===----------------------------------------------------------------------===//
1732
1733 //===----------------------------------------------------------------------===//
1734 // Base ENC_MUBUF for GFX6, GFX7, GFX10.
1735 //===----------------------------------------------------------------------===//
1736
1737 class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> :
1738     MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
1739   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1740   let Inst{12}    = ps.offen;
1741   let Inst{13}    = ps.idxen;
1742   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1743   let Inst{16}    = !if(ps.lds, 1, 0);
1744   let Inst{24-18} = op;
1745   let Inst{31-26} = 0x38;
1746   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1747   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1748   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1749   let Inst{54}    = !if(ps.has_slc, slc, ?);
1750   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1751   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1752 }
1753
1754 class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> :
1755     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> {
1756   let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value);
1757   let Inst{25} = op{7};
1758 }
1759
1760 class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> :
1761     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> {
1762   let Inst{15} = ps.addr64;
1763 }
1764
1765 //===----------------------------------------------------------------------===//
1766 // MUBUF - GFX10.
1767 //===----------------------------------------------------------------------===//
1768
1769 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
1770   multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName,
1771                                         string asmName> {
1772     def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> {
1773       MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName);
1774       let AsmString = asmName # ps.AsmOperands;
1775     }
1776   }
1777   multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> {
1778     def _BOTHEN_gfx10 :
1779       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1780     def _IDXEN_gfx10 :
1781       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1782     def _OFFEN_gfx10 :
1783       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1784     def _OFFSET_gfx10 :
1785       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1786   }
1787   multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> {
1788     def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1789                         MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">;
1790     def _OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1791                         MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">;
1792     def _IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1793                         MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">;
1794     def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1795                         MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">;
1796
1797     def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
1798                             MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">;
1799     def _LDS_OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
1800                             MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">;
1801     def _LDS_IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
1802                             MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">;
1803     def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
1804                             MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">;
1805   }
1806   multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> :
1807       MUBUF_Real_AllAddr_gfx10<op> {
1808     def _BOTHEN_RTN_gfx10 :
1809       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1810     def _IDXEN_RTN_gfx10 :
1811       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1812     def _OFFEN_RTN_gfx10 :
1813       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1814     def _OFFSET_RTN_gfx10 :
1815       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1816   }
1817 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
1818
1819 defm BUFFER_STORE_BYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x019>;
1820 defm BUFFER_STORE_SHORT_D16_HI    : MUBUF_Real_AllAddr_gfx10<0x01b>;
1821 defm BUFFER_LOAD_UBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x020>;
1822 defm BUFFER_LOAD_UBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x021>;
1823 defm BUFFER_LOAD_SBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x022>;
1824 defm BUFFER_LOAD_SBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x023>;
1825 defm BUFFER_LOAD_SHORT_D16        : MUBUF_Real_AllAddr_gfx10<0x024>;
1826 defm BUFFER_LOAD_SHORT_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x025>;
1827 // FIXME-GFX10: Add following instructions:
1828 //defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_gfx10<0x026>;
1829 //defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>;
1830 defm BUFFER_LOAD_FORMAT_D16_X     : MUBUF_Real_AllAddr_gfx10<0x080>;
1831 defm BUFFER_LOAD_FORMAT_D16_XY    : MUBUF_Real_AllAddr_gfx10<0x081>;
1832 defm BUFFER_LOAD_FORMAT_D16_XYZ   : MUBUF_Real_AllAddr_gfx10<0x082>;
1833 defm BUFFER_LOAD_FORMAT_D16_XYZW  : MUBUF_Real_AllAddr_gfx10<0x083>;
1834 defm BUFFER_STORE_FORMAT_D16_X    : MUBUF_Real_AllAddr_gfx10<0x084>;
1835 defm BUFFER_STORE_FORMAT_D16_XY   : MUBUF_Real_AllAddr_gfx10<0x085>;
1836 defm BUFFER_STORE_FORMAT_D16_XYZ  : MUBUF_Real_AllAddr_gfx10<0x086>;
1837 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>;
1838
1839 def BUFFER_GL0_INV_gfx10 :
1840   MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>;
1841 def BUFFER_GL1_INV_gfx10 :
1842   MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>;
1843
1844 //===----------------------------------------------------------------------===//
1845 // MUBUF - GFX6, GFX7, GFX10.
1846 //===----------------------------------------------------------------------===//
1847
1848 let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in {
1849   multiclass MUBUF_Real_gfx6<bits<8> op> {
1850     def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
1851   }
1852 } // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6"
1853
1854 let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {
1855   multiclass MUBUF_Real_gfx7<bits<8> op> {
1856     def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
1857   }
1858 } // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"
1859
1860 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
1861   multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> {
1862     def _ADDR64_gfx6_gfx7 :
1863       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>;
1864     def _BOTHEN_gfx6_gfx7 :
1865       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1866     def _IDXEN_gfx6_gfx7 :
1867       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1868     def _OFFEN_gfx6_gfx7 :
1869       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1870     def _OFFSET_gfx6_gfx7 :
1871       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1872   }
1873   multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> {
1874     def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1875                             MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">;
1876     def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>,
1877                             MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">;
1878     def _OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1879                             MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">;
1880     def _IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1881                             MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">;
1882     def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1883                             MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">;
1884
1885     def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
1886                                 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">;
1887     def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>,
1888                                 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">;
1889     def _LDS_OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
1890                                 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">;
1891     def _LDS_IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
1892                                 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">;
1893     def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
1894                                 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">;
1895   }
1896   multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> :
1897       MUBUF_Real_AllAddr_gfx6_gfx7<op> {
1898     def _ADDR64_RTN_gfx6_gfx7 :
1899       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>;
1900     def _BOTHEN_RTN_gfx6_gfx7 :
1901       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1902     def _IDXEN_RTN_gfx6_gfx7 :
1903       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1904     def _OFFEN_RTN_gfx6_gfx7 :
1905       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1906     def _OFFSET_RTN_gfx6_gfx7 :
1907       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1908   }
1909 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
1910
1911 multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> :
1912   MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>;
1913
1914 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> :
1915   MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>;
1916
1917 multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> :
1918   MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>;
1919
1920 // FIXME-GFX6: Following instructions are available only on GFX6.
1921 //defm BUFFER_ATOMIC_RSUB         : MUBUF_Real_Atomics_gfx6 <0x034>;
1922 //defm BUFFER_ATOMIC_RSUB_X2      : MUBUF_Real_Atomics_gfx6 <0x054>;
1923
1924 defm BUFFER_LOAD_FORMAT_X     : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>;
1925 defm BUFFER_LOAD_FORMAT_XY    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
1926 defm BUFFER_LOAD_FORMAT_XYZ   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
1927 defm BUFFER_LOAD_FORMAT_XYZW  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
1928 defm BUFFER_STORE_FORMAT_X    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
1929 defm BUFFER_STORE_FORMAT_XY   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
1930 defm BUFFER_STORE_FORMAT_XYZ  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
1931 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
1932 defm BUFFER_LOAD_UBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>;
1933 defm BUFFER_LOAD_SBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>;
1934 defm BUFFER_LOAD_USHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>;
1935 defm BUFFER_LOAD_SSHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>;
1936 defm BUFFER_LOAD_DWORD        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>;
1937 defm BUFFER_LOAD_DWORDX2      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>;
1938 defm BUFFER_LOAD_DWORDX4      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>;
1939 defm BUFFER_LOAD_DWORDX3      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>;
1940 defm BUFFER_STORE_BYTE        : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>;
1941 defm BUFFER_STORE_SHORT       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>;
1942 defm BUFFER_STORE_DWORD       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>;
1943 defm BUFFER_STORE_DWORDX2     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>;
1944 defm BUFFER_STORE_DWORDX4     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>;
1945 defm BUFFER_STORE_DWORDX3     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>;
1946
1947 defm BUFFER_ATOMIC_SWAP        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>;
1948 defm BUFFER_ATOMIC_CMPSWAP     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>;
1949 defm BUFFER_ATOMIC_ADD         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>;
1950 defm BUFFER_ATOMIC_SUB         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>;
1951 defm BUFFER_ATOMIC_SMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>;
1952 defm BUFFER_ATOMIC_UMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>;
1953 defm BUFFER_ATOMIC_SMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>;
1954 defm BUFFER_ATOMIC_UMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>;
1955 defm BUFFER_ATOMIC_AND         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>;
1956 defm BUFFER_ATOMIC_OR          : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>;
1957 defm BUFFER_ATOMIC_XOR         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>;
1958 defm BUFFER_ATOMIC_INC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>;
1959 defm BUFFER_ATOMIC_DEC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>;
1960 // FIXME-GFX6-GFX7-GFX10: Add following instructions:
1961 //defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>;
1962 //defm BUFFER_ATOMIC_FMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>;
1963 //defm BUFFER_ATOMIC_FMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>;
1964 defm BUFFER_ATOMIC_SWAP_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>;
1965 defm BUFFER_ATOMIC_CMPSWAP_X2  : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>;
1966 defm BUFFER_ATOMIC_ADD_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>;
1967 defm BUFFER_ATOMIC_SUB_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>;
1968 defm BUFFER_ATOMIC_SMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>;
1969 defm BUFFER_ATOMIC_UMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>;
1970 defm BUFFER_ATOMIC_SMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>;
1971 defm BUFFER_ATOMIC_UMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>;
1972 defm BUFFER_ATOMIC_AND_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>;
1973 defm BUFFER_ATOMIC_OR_X2       : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>;
1974 defm BUFFER_ATOMIC_XOR_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>;
1975 defm BUFFER_ATOMIC_INC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>;
1976 defm BUFFER_ATOMIC_DEC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>;
1977 // FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7.
1978 // FIXME-GFX6-GFX7-GFX10: Add following instructions:
1979 //defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>;
1980 //defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>;
1981 //defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>;
1982
1983 defm BUFFER_WBINVL1_SC        : MUBUF_Real_gfx6<0x070>;
1984 defm BUFFER_WBINVL1_VOL       : MUBUF_Real_gfx7<0x070>;
1985 def  BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>;
1986
1987 //===----------------------------------------------------------------------===//
1988 // Base ENC_MTBUF for GFX6, GFX7, GFX10.
1989 //===----------------------------------------------------------------------===//
1990
1991 class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> :
1992     MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
1993   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1994   let Inst{12}    = ps.offen;
1995   let Inst{13}    = ps.idxen;
1996   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1997   let Inst{18-16} = op;
1998   let Inst{31-26} = 0x3a; //encoding
1999   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2000   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2001   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2002   let Inst{54}    = !if(ps.has_slc, slc, ?);
2003   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2004   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2005 }
2006
2007 //===----------------------------------------------------------------------===//
2008 // MTBUF - GFX10.
2009 //===----------------------------------------------------------------------===//
2010
2011 class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> :
2012     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> {
2013   let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value);
2014   let Inst{25-19} = format;
2015   let Inst{53} = op{3};
2016 }
2017
2018 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
2019   multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> {
2020     def _BOTHEN_gfx10 :
2021       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2022     def _IDXEN_gfx10 :
2023       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2024     def _OFFEN_gfx10 :
2025       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2026     def _OFFSET_gfx10 :
2027       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2028   }
2029 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
2030
2031 defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_gfx10<0x008>;
2032 defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_gfx10<0x009>;
2033 defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_gfx10<0x00a>;
2034 defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_gfx10<0x00b>;
2035 defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_gfx10<0x00c>;
2036 defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_gfx10<0x00d>;
2037 defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_gfx10<0x00e>;
2038 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>;
2039
2040 //===----------------------------------------------------------------------===//
2041 // MTBUF - GFX6, GFX7, GFX10.
2042 //===----------------------------------------------------------------------===//
2043
2044 class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> :
2045     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> {
2046   let Inst{15} = ps.addr64;
2047   let Inst{22-19} = dfmt;
2048   let Inst{25-23} = nfmt;
2049 }
2050
2051 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
2052   multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> {
2053     def _ADDR64_gfx6_gfx7 :
2054       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>;
2055     def _BOTHEN_gfx6_gfx7 :
2056       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2057     def _IDXEN_gfx6_gfx7 :
2058       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2059     def _OFFEN_gfx6_gfx7 :
2060       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2061     def _OFFSET_gfx6_gfx7 :
2062       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2063   }
2064 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
2065
2066 multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> :
2067   MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>;
2068
2069 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>;
2070 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
2071 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
2072 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
2073 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
2074 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
2075 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
2076 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
2077
2078 //===----------------------------------------------------------------------===//
2079 // GFX8, GFX9 (VI).
2080 //===----------------------------------------------------------------------===//
2081
2082 class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> :
2083   MUBUF_Real<ps>,
2084   Enc64,
2085   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
2086   let AssemblerPredicate = isGFX8GFX9;
2087   let DecoderNamespace = "GFX8";
2088
2089   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2090   let Inst{12}    = ps.offen;
2091   let Inst{13}    = ps.idxen;
2092   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2093   let Inst{16}    = !if(ps.lds, 1, 0);
2094   let Inst{17}    = !if(ps.has_slc, slc, ?);
2095   let Inst{24-18} = op;
2096   let Inst{31-26} = 0x38; //encoding
2097   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2098   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2099   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2100   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2101   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2102 }
2103
2104 multiclass MUBUF_Real_AllAddr_vi<bits<7> op> {
2105   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2106   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2107   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2108   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2109 }
2110
2111 multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> {
2112
2113   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
2114                    MUBUFLdsTable<0, NAME # "_OFFSET_vi">;
2115   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
2116                    MUBUFLdsTable<0, NAME # "_OFFEN_vi">;
2117   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
2118                    MUBUFLdsTable<0, NAME # "_IDXEN_vi">;
2119   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
2120                    MUBUFLdsTable<0, NAME # "_BOTHEN_vi">;
2121
2122   def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
2123                        MUBUFLdsTable<1, NAME # "_OFFSET_vi">;
2124   def _LDS_OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
2125                        MUBUFLdsTable<1, NAME # "_OFFEN_vi">;
2126   def _LDS_IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
2127                        MUBUFLdsTable<1, NAME # "_IDXEN_vi">;
2128   def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
2129                        MUBUFLdsTable<1, NAME # "_BOTHEN_vi">;
2130 }
2131
2132 class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> :
2133   MUBUF_Real<ps>,
2134   Enc64,
2135   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2136   let AssemblerPredicate=HasUnpackedD16VMem;
2137   let DecoderNamespace="GFX80_UNPACKED";
2138
2139   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2140   let Inst{12}    = ps.offen;
2141   let Inst{13}    = ps.idxen;
2142   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2143   let Inst{16}    = !if(ps.lds, 1, 0);
2144   let Inst{17}    = !if(ps.has_slc, slc, ?);
2145   let Inst{24-18} = op;
2146   let Inst{31-26} = 0x38; //encoding
2147   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2148   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2149   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2150   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2151   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2152 }
2153
2154 multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> {
2155   def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2156   def _OFFEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2157   def _IDXEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2158   def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2159 }
2160
2161 multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
2162   MUBUF_Real_AllAddr_vi<op> {
2163   def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
2164   def _OFFEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
2165   def _IDXEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
2166   def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
2167 }
2168
2169 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_Lds_vi <0x00>;
2170 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_vi <0x01>;
2171 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_vi <0x02>;
2172 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_vi <0x03>;
2173 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_vi <0x04>;
2174 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_vi <0x05>;
2175 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_vi <0x06>;
2176 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_vi <0x07>;
2177 let SubtargetPredicate = HasUnpackedD16VMem in {
2178   defm BUFFER_LOAD_FORMAT_D16_X_gfx80       : MUBUF_Real_AllAddr_gfx80 <0x08>;
2179   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x09>;
2180   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0a>;
2181   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0b>;
2182   defm BUFFER_STORE_FORMAT_D16_X_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x0c>;
2183   defm BUFFER_STORE_FORMAT_D16_XY_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0d>;
2184   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0e>;
2185   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80   : MUBUF_Real_AllAddr_gfx80 <0x0f>;
2186 } // End HasUnpackedD16VMem.
2187 let SubtargetPredicate = HasPackedD16VMem in {
2188   defm BUFFER_LOAD_FORMAT_D16_X       : MUBUF_Real_AllAddr_vi <0x08>;
2189   defm BUFFER_LOAD_FORMAT_D16_XY      : MUBUF_Real_AllAddr_vi <0x09>;
2190   defm BUFFER_LOAD_FORMAT_D16_XYZ     : MUBUF_Real_AllAddr_vi <0x0a>;
2191   defm BUFFER_LOAD_FORMAT_D16_XYZW    : MUBUF_Real_AllAddr_vi <0x0b>;
2192   defm BUFFER_STORE_FORMAT_D16_X      : MUBUF_Real_AllAddr_vi <0x0c>;
2193   defm BUFFER_STORE_FORMAT_D16_XY     : MUBUF_Real_AllAddr_vi <0x0d>;
2194   defm BUFFER_STORE_FORMAT_D16_XYZ    : MUBUF_Real_AllAddr_vi <0x0e>;
2195   defm BUFFER_STORE_FORMAT_D16_XYZW   : MUBUF_Real_AllAddr_vi <0x0f>;
2196 } // End HasPackedD16VMem.
2197 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x10>;
2198 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x11>;
2199 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_Lds_vi <0x12>;
2200 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_Lds_vi <0x13>;
2201 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_Lds_vi <0x14>;
2202 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_Lds_vi <0x15>;
2203 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_Lds_vi <0x16>;
2204 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_Lds_vi <0x17>;
2205 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_vi <0x18>;
2206 defm BUFFER_STORE_BYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x19>;
2207 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_vi <0x1a>;
2208 defm BUFFER_STORE_SHORT_D16_HI  : MUBUF_Real_AllAddr_vi <0x1b>;
2209 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_vi <0x1c>;
2210 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_vi <0x1d>;
2211 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_vi <0x1e>;
2212 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_vi <0x1f>;
2213
2214 defm BUFFER_LOAD_UBYTE_D16      : MUBUF_Real_AllAddr_vi <0x20>;
2215 defm BUFFER_LOAD_UBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x21>;
2216 defm BUFFER_LOAD_SBYTE_D16      : MUBUF_Real_AllAddr_vi <0x22>;
2217 defm BUFFER_LOAD_SBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x23>;
2218 defm BUFFER_LOAD_SHORT_D16      : MUBUF_Real_AllAddr_vi <0x24>;
2219 defm BUFFER_LOAD_SHORT_D16_HI   : MUBUF_Real_AllAddr_vi <0x25>;
2220
2221 defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_vi <0x26>;
2222 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>;
2223
2224 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_vi <0x40>;
2225 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_vi <0x41>;
2226 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_vi <0x42>;
2227 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_vi <0x43>;
2228 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_vi <0x44>;
2229 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_vi <0x45>;
2230 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_vi <0x46>;
2231 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_vi <0x47>;
2232 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_vi <0x48>;
2233 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_vi <0x49>;
2234 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_vi <0x4a>;
2235 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_vi <0x4b>;
2236 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_vi <0x4c>;
2237
2238 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_vi <0x60>;
2239 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_vi <0x61>;
2240 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_vi <0x62>;
2241 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_vi <0x63>;
2242 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_vi <0x64>;
2243 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_vi <0x65>;
2244 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_vi <0x66>;
2245 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_vi <0x67>;
2246 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_vi <0x68>;
2247 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_vi <0x69>;
2248 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_vi <0x6a>;
2249 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_vi <0x6b>;
2250 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_vi <0x6c>;
2251
2252 def BUFFER_STORE_LDS_DWORD_vi   : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>;
2253
2254 def BUFFER_WBINVL1_vi           : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>;
2255 def BUFFER_WBINVL1_VOL_vi       : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>;
2256
2257 let SubtargetPredicate = HasAtomicFaddInsts in {
2258
2259 defm BUFFER_ATOMIC_ADD_F32    : MUBUF_Real_AllAddr_vi <0x4d>;
2260 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_AllAddr_vi <0x4e>;
2261
2262 } // End SubtargetPredicate = HasAtomicFaddInsts
2263
2264 class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
2265   MTBUF_Real<ps>,
2266   Enc64,
2267   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
2268   let AssemblerPredicate = isGFX8GFX9;
2269   let DecoderNamespace = "GFX8";
2270
2271   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2272   let Inst{12}    = ps.offen;
2273   let Inst{13}    = ps.idxen;
2274   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2275   let Inst{18-15} = op;
2276   let Inst{22-19} = dfmt;
2277   let Inst{25-23} = nfmt;
2278   let Inst{31-26} = 0x3a; //encoding
2279   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2280   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2281   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2282   let Inst{54}    = !if(ps.has_slc, slc, ?);
2283   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2284   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2285 }
2286
2287 multiclass MTBUF_Real_AllAddr_vi<bits<4> op> {
2288   def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2289   def _OFFEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2290   def _IDXEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2291   def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2292 }
2293
2294 class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> :
2295   MTBUF_Real<ps>,
2296   Enc64,
2297   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2298   let AssemblerPredicate=HasUnpackedD16VMem;
2299   let DecoderNamespace="GFX80_UNPACKED";
2300
2301   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2302   let Inst{12}    = ps.offen;
2303   let Inst{13}    = ps.idxen;
2304   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2305   let Inst{18-15} = op;
2306   let Inst{22-19} = dfmt;
2307   let Inst{25-23} = nfmt;
2308   let Inst{31-26} = 0x3a; //encoding
2309   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2310   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2311   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2312   let Inst{54}    = !if(ps.has_slc, slc, ?);
2313   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2314   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2315 }
2316
2317 multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> {
2318   def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2319   def _OFFEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2320   def _IDXEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2321   def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2322 }
2323
2324 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_vi <0x00>;
2325 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_vi <0x01>;
2326 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_vi <0x02>;
2327 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_vi <0x03>;
2328 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_vi <0x04>;
2329 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_vi <0x05>;
2330 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_vi <0x06>;
2331 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>;
2332 let SubtargetPredicate = HasUnpackedD16VMem in {
2333   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Real_AllAddr_gfx80 <0x08>;
2334   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x09>;
2335   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0a>;
2336   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0b>;
2337   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x0c>;
2338   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0d>;
2339   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0e>;
2340   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>;
2341 } // End HasUnpackedD16VMem.
2342 let SubtargetPredicate = HasPackedD16VMem in {
2343   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_vi <0x08>;
2344   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_vi <0x09>;
2345   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_vi <0x0a>;
2346   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_vi <0x0b>;
2347   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_vi <0x0c>;
2348   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_vi <0x0d>;
2349   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_vi <0x0e>;
2350   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>;
2351 } // End HasUnpackedD16VMem.
2352
2353 def MUBUFInfoTable : GenericTable {
2354   let FilterClass = "MUBUF_Pseudo";
2355   let CppTypeName = "MUBUFInfo";
2356   let Fields = ["Opcode", "BaseOpcode", "dwords", "has_vaddr", "has_srsrc", "has_soffset"];
2357
2358   let PrimaryKey = ["Opcode"];
2359   let PrimaryKeyName = "getMUBUFOpcodeHelper";
2360 }
2361
2362 def getMUBUFInfoFromOpcode : SearchIndex {
2363   let Table = MUBUFInfoTable;
2364   let Key = ["Opcode"];
2365 }
2366
2367 def getMUBUFInfoFromBaseOpcodeAndDwords : SearchIndex {
2368   let Table = MUBUFInfoTable;
2369   let Key = ["BaseOpcode", "dwords"];
2370 }