contrib/llvm-project/llvm/lib/Target/AMDGPU/BUFInstructions.td

   1 //===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">;
  10 def MUBUFAddr64 : ComplexPattern<i64, 9, "SelectMUBUFAddr64">;
  11 def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">;
  12
  13 def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>;
  14 def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>;
  15
  16 def MUBUFOffset : ComplexPattern<i64, 8, "SelectMUBUFOffset">;
  17 def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
  18 def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">;
  19
  20 def BUFAddrKind {
  21   int Offset = 0;
  22   int OffEn  = 1;
  23   int IdxEn  = 2;
  24   int BothEn = 3;
  25   int Addr64 = 4;
  26 }
  27
  28 class getAddrName<int addrKind> {
  29   string ret =
  30     !if(!eq(addrKind, BUFAddrKind.Offset), "offset",
  31     !if(!eq(addrKind, BUFAddrKind.OffEn),  "offen",
  32     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "idxen",
  33     !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",
  34     !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",
  35     "")))));
  36 }
  37
  38 class MUBUFAddr64Table <bit is_addr64, string Name> {
  39   bit IsAddr64 = is_addr64;
  40   string OpName = Name;
  41 }
  42
  43 class MUBUFLdsTable <bit is_lds, string Name> {
  44   bit IsLds = is_lds;
  45   string OpName = Name;
  46 }
  47
  48 class MTBUFAddr64Table <bit is_addr64, string Name> {
  49   bit IsAddr64 = is_addr64;
  50   string OpName = Name;
  51 }
  52
  53 //===----------------------------------------------------------------------===//
  54 // MTBUF classes
  55 //===----------------------------------------------------------------------===//
  56
  57 class MTBUFGetBaseOpcode<string Op> {
  58   string ret = !subst("FORMAT_XY", "FORMAT_X",
  59     !subst("FORMAT_XYZ", "FORMAT_X",
  60     !subst("FORMAT_XYZW", "FORMAT_X", Op)));
  61 }
  62
  63 class getMTBUFElements<string Op> {
  64   int ret = 1;
  65 }
  66
  67
  68 class MTBUF_Pseudo <string opName, dag outs, dag ins,
  69                     string asmOps, list<dag> pattern=[]> :
  70   InstSI<outs, ins, "", pattern>,
  71   SIMCInstr<opName, SIEncodingFamily.NONE> {
  72
  73   let isPseudo = 1;
  74   let isCodeGenOnly = 1;
  75   let Size = 8;
  76   let UseNamedOperandTable = 1;
  77
  78   string Mnemonic = opName;
  79   string AsmOperands = asmOps;
  80
  81   Instruction Opcode = !cast<Instruction>(NAME);
  82   Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret);
  83
  84   let VM_CNT = 1;
  85   let EXP_CNT = 1;
  86   let MTBUF = 1;
  87   let Uses = [EXEC];
  88   let hasSideEffects = 0;
  89   let SchedRW = [WriteVMEM];
  90
  91   let AsmMatchConverter = "cvtMtbuf";
  92
  93   bits<1> offen       = 0;
  94   bits<1> idxen       = 0;
  95   bits<1> addr64      = 0;
  96   bits<1> has_vdata   = 1;
  97   bits<1> has_vaddr   = 1;
  98   bits<1> has_glc     = 1;
  99   bits<1> has_dlc     = 1;
 100   bits<1> glc_value   = 0; // the value for glc if no such operand
 101   bits<1> dlc_value   = 0; // the value for dlc if no such operand
 102   bits<1> has_srsrc   = 1;
 103   bits<1> has_soffset = 1;
 104   bits<1> has_offset  = 1;
 105   bits<1> has_slc     = 1;
 106   bits<1> has_tfe     = 1;
 107   bits<4> elements    = 0;
 108 }
 109
 110 class MTBUF_Real <MTBUF_Pseudo ps> :
 111   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 112
 113   let isPseudo = 0;
 114   let isCodeGenOnly = 0;
 115
 116   // copy relevant pseudo op flags
 117   let SubtargetPredicate = ps.SubtargetPredicate;
 118   let AsmMatchConverter  = ps.AsmMatchConverter;
 119   let Constraints        = ps.Constraints;
 120   let DisableEncoding    = ps.DisableEncoding;
 121   let TSFlags            = ps.TSFlags;
 122
 123   bits<12> offset;
 124   bits<1>  glc;
 125   bits<1>  dlc;
 126   bits<7>  format;
 127   bits<8>  vaddr;
 128   bits<8>  vdata;
 129   bits<7>  srsrc;
 130   bits<1>  slc;
 131   bits<1>  tfe;
 132   bits<8>  soffset;
 133
 134   bits<4> dfmt = format{3-0};
 135   bits<3> nfmt = format{6-4};
 136 }
 137
 138 class getMTBUFInsDA<list<RegisterClass> vdataList,
 139                     list<RegisterClass> vaddrList=[]> {
 140   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 141   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 142   dag InsNoData = !if(!empty(vaddrList),
 143     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 144          offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz),
 145     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 146          offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz)
 147   );
 148   dag InsData = !if(!empty(vaddrList),
 149     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 150          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
 151          SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz),
 152     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 153          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
 154          SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz)
 155   );
 156   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 157 }
 158
 159 class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
 160   dag ret =
 161     !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret,
 162     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 163     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 164     !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 165     !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 166     (ins))))));
 167 }
 168
 169 class getMTBUFAsmOps<int addrKind> {
 170   string Pfx =
 171     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $format, $soffset",
 172     !if(!eq(addrKind, BUFAddrKind.OffEn),
 173             "$vaddr, $srsrc, $format, $soffset offen",
 174     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 175             "$vaddr, $srsrc, $format, $soffset idxen",
 176     !if(!eq(addrKind, BUFAddrKind.BothEn),
 177             "$vaddr, $srsrc, $format, $soffset idxen offen",
 178     !if(!eq(addrKind, BUFAddrKind.Addr64),
 179             "$vaddr, $srsrc, $format, $soffset addr64",
 180     "")))));
 181   string ret = Pfx # "$offset";
 182 }
 183
 184 class MTBUF_SetupAddr<int addrKind> {
 185   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 186                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 187
 188   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 189                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 190
 191   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 192
 193   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 194 }
 195
 196 class MTBUF_Load_Pseudo <string opName,
 197                          int addrKind,
 198                          RegisterClass vdataClass,
 199                          int elems,
 200                          list<dag> pattern=[],
 201                          // Workaround bug bz30254
 202                          int addrKindCopy = addrKind>
 203   : MTBUF_Pseudo<opName,
 204                  (outs vdataClass:$vdata),
 205                  getMTBUFIns<addrKindCopy>.ret,
 206                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz",
 207                  pattern>,
 208     MTBUF_SetupAddr<addrKindCopy> {
 209   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 210   let mayLoad = 1;
 211   let mayStore = 0;
 212   let elements = elems;
 213 }
 214
 215 multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 216                               int elems, ValueType load_vt = i32,
 217                               SDPatternOperator ld = null_frag> {
 218
 219   def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems,
 220     [(set load_vt:$vdata,
 221      (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format,
 222                       i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>,
 223     MTBUFAddr64Table<0, NAME>;
 224
 225   def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems,
 226     [(set load_vt:$vdata,
 227      (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset,
 228                       i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>,
 229     MTBUFAddr64Table<1, NAME>;
 230
 231   def _OFFEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 232   def _IDXEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 233   def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 234
 235   let DisableWQM = 1 in {
 236     def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>;
 237     def _OFFEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 238     def _IDXEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 239     def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 240   }
 241 }
 242
 243 class MTBUF_Store_Pseudo <string opName,
 244                           int addrKind,
 245                           RegisterClass vdataClass,
 246                           int elems,
 247                           list<dag> pattern=[],
 248                           // Workaround bug bz30254
 249                           int addrKindCopy = addrKind,
 250                           RegisterClass vdataClassCopy = vdataClass>
 251   : MTBUF_Pseudo<opName,
 252                  (outs),
 253                  getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 254                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz",
 255                  pattern>,
 256     MTBUF_SetupAddr<addrKindCopy> {
 257   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 258   let mayLoad = 0;
 259   let mayStore = 1;
 260   let elements = elems;
 261 }
 262
 263 multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 264                                int elems, ValueType store_vt = i32,
 265                                SDPatternOperator st = null_frag> {
 266
 267   def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems,
 268     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 269                                        i16:$offset, i8:$format, i1:$glc,
 270                                        i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
 271     MTBUFAddr64Table<0, NAME>;
 272
 273   def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems,
 274     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 275                                        i16:$offset, i8:$format, i1:$glc,
 276                                        i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
 277     MTBUFAddr64Table<1, NAME>;
 278
 279   def _OFFEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 280   def _IDXEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 281   def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 282
 283   let DisableWQM = 1 in {
 284     def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>;
 285     def _OFFEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 286     def _IDXEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 287     def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 288   }
 289 }
 290
 291
 292 //===----------------------------------------------------------------------===//
 293 // MUBUF classes
 294 //===----------------------------------------------------------------------===//
 295
 296 class MUBUFGetBaseOpcode<string Op> {
 297   string ret = !subst("DWORDX2", "DWORD",
 298     !subst("DWORDX3", "DWORD",
 299     !subst("DWORDX4", "DWORD", Op)));
 300 }
 301
 302 class MUBUF_Pseudo <string opName, dag outs, dag ins,
 303                     string asmOps, list<dag> pattern=[]> :
 304   InstSI<outs, ins, "", pattern>,
 305   SIMCInstr<opName, SIEncodingFamily.NONE> {
 306
 307   let isPseudo = 1;
 308   let isCodeGenOnly = 1;
 309   let Size = 8;
 310   let UseNamedOperandTable = 1;
 311
 312   string Mnemonic = opName;
 313   string AsmOperands = asmOps;
 314
 315   Instruction Opcode = !cast<Instruction>(NAME);
 316   Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret);
 317
 318   let VM_CNT = 1;
 319   let EXP_CNT = 1;
 320   let MUBUF = 1;
 321   let Uses = [EXEC];
 322   let hasSideEffects = 0;
 323   let SchedRW = [WriteVMEM];
 324
 325   let AsmMatchConverter = "cvtMubuf";
 326
 327   bits<1> offen       = 0;
 328   bits<1> idxen       = 0;
 329   bits<1> addr64      = 0;
 330   bits<1> lds         = 0;
 331   bits<1> has_vdata   = 1;
 332   bits<1> has_vaddr   = 1;
 333   bits<1> has_glc     = 1;
 334   bits<1> has_dlc     = 1;
 335   bits<1> glc_value   = 0; // the value for glc if no such operand
 336   bits<1> dlc_value   = 0; // the value for dlc if no such operand
 337   bits<1> has_srsrc   = 1;
 338   bits<1> has_soffset = 1;
 339   bits<1> has_offset  = 1;
 340   bits<1> has_slc     = 1;
 341   bits<1> has_tfe     = 1;
 342   bits<4> elements    = 0;
 343 }
 344
 345 class MUBUF_Real <MUBUF_Pseudo ps> :
 346   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 347
 348   let isPseudo = 0;
 349   let isCodeGenOnly = 0;
 350
 351   // copy relevant pseudo op flags
 352   let SubtargetPredicate = ps.SubtargetPredicate;
 353   let AsmMatchConverter  = ps.AsmMatchConverter;
 354   let Constraints        = ps.Constraints;
 355   let DisableEncoding    = ps.DisableEncoding;
 356   let TSFlags            = ps.TSFlags;
 357
 358   bits<12> offset;
 359   bits<1>  glc;
 360   bits<1>  dlc;
 361   bits<8>  vaddr;
 362   bits<8>  vdata;
 363   bits<7>  srsrc;
 364   bits<1>  slc;
 365   bits<1>  tfe;
 366   bits<8>  soffset;
 367 }
 368
 369
 370 // For cache invalidation instructions.
 371 class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> :
 372   MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {
 373
 374   let AsmMatchConverter = "";
 375
 376   let hasSideEffects = 1;
 377   let mayLoad = 0;
 378   let mayStore = 0;
 379
 380   // Set everything to 0.
 381   let offen       = 0;
 382   let idxen       = 0;
 383   let addr64      = 0;
 384   let has_vdata   = 0;
 385   let has_vaddr   = 0;
 386   let has_glc     = 0;
 387   let has_dlc     = 0;
 388   let glc_value   = 0;
 389   let dlc_value   = 0;
 390   let has_srsrc   = 0;
 391   let has_soffset = 0;
 392   let has_offset  = 0;
 393   let has_slc     = 0;
 394   let has_tfe     = 0;
 395 }
 396
 397 class getMUBUFInsDA<list<RegisterClass> vdataList,
 398                     list<RegisterClass> vaddrList=[],
 399                     bit isLds = 0> {
 400   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 401   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 402   dag InsNoData = !if(!empty(vaddrList),
 403     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 404          offset:$offset, GLC:$glc, SLC:$slc),
 405     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 406          offset:$offset, GLC:$glc, SLC:$slc)
 407   );
 408   dag InsData = !if(!empty(vaddrList),
 409     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 410          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc),
 411     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 412          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc)
 413   );
 414   dag ret = !con(
 415               !if(!empty(vdataList), InsNoData, InsData),
 416               !if(isLds, (ins DLC:$dlc, SWZ:$swz), (ins TFE:$tfe, DLC:$dlc,SWZ:$swz))
 417              );
 418 }
 419
 420 class getMUBUFElements<ValueType vt> {
 421   // eq does not support ValueType for some reason.
 422   string vtAsStr = !cast<string>(vt);
 423
 424   int ret =
 425     !if(!eq(vtAsStr, "f16"), 1,
 426       !if(!eq(vtAsStr, "v2f16"), 2,
 427         !if(!eq(vtAsStr, "v3f16"), 3,
 428           !if(!eq(vtAsStr, "v4f16"), 4,
 429             !if(!eq(vt.Size, 32), 1,
 430               !if(!eq(vt.Size, 64), 2,
 431                 !if(!eq(vt.Size, 96), 3,
 432                   !if(!eq(vt.Size, 128), 4, 0)
 433                 )
 434               )
 435             )
 436           )
 437         )
 438       )
 439     );
 440 }
 441
 442 class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> {
 443   dag ret =
 444     !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret,
 445     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 446     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 447     !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 448     !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 449     (ins))))));
 450 }
 451
 452 class getMUBUFAsmOps<int addrKind> {
 453   string Pfx =
 454     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",
 455     !if(!eq(addrKind, BUFAddrKind.OffEn),  "$vaddr, $srsrc, $soffset offen",
 456     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "$vaddr, $srsrc, $soffset idxen",
 457     !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",
 458     !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",
 459     "")))));
 460   string ret = Pfx # "$offset";
 461 }
 462
 463 class MUBUF_SetupAddr<int addrKind> {
 464   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 465                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 466
 467   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 468                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 469
 470   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 471
 472   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 473 }
 474
 475 class MUBUF_Load_Pseudo <string opName,
 476                          int addrKind,
 477                          ValueType vdata_vt,
 478                          bit HasTiedDest = 0,
 479                          bit isLds = 0,
 480                          list<dag> pattern=[],
 481                          // Workaround bug bz30254
 482                          int addrKindCopy = addrKind>
 483   : MUBUF_Pseudo<opName,
 484                  (outs getVregSrcForVT<vdata_vt>.ret:$vdata),
 485                  !con(getMUBUFIns<addrKindCopy, [], isLds>.ret,
 486                       !if(HasTiedDest, (ins getVregSrcForVT<vdata_vt>.ret:$vdata_in), (ins))),
 487                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" #
 488                    !if(isLds, " lds", "$tfe") # "$dlc" # "$swz",
 489                  pattern>,
 490     MUBUF_SetupAddr<addrKindCopy> {
 491   let PseudoInstr = opName # !if(isLds, "_lds", "") #
 492                     "_" # getAddrName<addrKindCopy>.ret;
 493   let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf");
 494
 495   let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", "");
 496   let mayLoad = 1;
 497   let mayStore = 0;
 498   let maybeAtomic = 1;
 499   let Uses = !if(isLds, [EXEC, M0], [EXEC]);
 500   let has_tfe = !if(isLds, 0, 1);
 501   let lds = isLds;
 502   let elements = getMUBUFElements<vdata_vt>.ret;
 503 }
 504
 505 class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat <
 506   (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
 507   (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))
 508 >;
 509
 510 class MUBUF_Addr64_Load_Pat <Instruction inst,
 511                             ValueType load_vt = i32,
 512                             SDPatternOperator ld = null_frag> : Pat <
 513   (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
 514   (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))
 515 >;
 516
 517 multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> {
 518   def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>;
 519   def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>;
 520 }
 521
 522
 523 // FIXME: tfe can't be an operand because it requires a separate
 524 // opcode because it needs an N+1 register class dest register.
 525 multiclass MUBUF_Pseudo_Loads<string opName,
 526                               ValueType load_vt = i32,
 527                               SDPatternOperator ld = null_frag,
 528                               bit TiedDest = 0,
 529                               bit isLds = 0> {
 530
 531   def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>,
 532     MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>;
 533
 534   def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, load_vt, TiedDest, isLds>,
 535     MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>;
 536
 537   def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>;
 538   def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>;
 539   def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>;
 540
 541   let DisableWQM = 1 in {
 542     def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>;
 543     def _OFFEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>;
 544     def _IDXEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>;
 545     def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>;
 546   }
 547 }
 548
 549 multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32,
 550                                   SDPatternOperator ld_nolds = null_frag,
 551                                   SDPatternOperator ld_lds = null_frag> {
 552   defm NAME : MUBUF_Pseudo_Loads<opName, load_vt, ld_nolds>;
 553   defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, ld_lds, 0, 1>;
 554 }
 555
 556 class MUBUF_Store_Pseudo <string opName,
 557                           int addrKind,
 558                           ValueType store_vt,
 559                           list<dag> pattern=[],
 560                           // Workaround bug bz30254
 561                           int addrKindCopy = addrKind>
 562   : MUBUF_Pseudo<opName,
 563                  (outs),
 564                  getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret,
 565                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz",
 566                  pattern>,
 567     MUBUF_SetupAddr<addrKindCopy> {
 568   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 569   let mayLoad = 0;
 570   let mayStore = 1;
 571   let maybeAtomic = 1;
 572   let elements = getMUBUFElements<store_vt>.ret;
 573 }
 574
 575 multiclass MUBUF_Pseudo_Stores<string opName,
 576                                ValueType store_vt = i32,
 577                                SDPatternOperator st = null_frag> {
 578
 579   def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt,
 580     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 581                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
 582     MUBUFAddr64Table<0, NAME>;
 583
 584   def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, store_vt,
 585     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 586                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
 587     MUBUFAddr64Table<1, NAME>;
 588
 589   def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>;
 590   def _IDXEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>;
 591   def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>;
 592
 593   let DisableWQM = 1 in {
 594     def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt>;
 595     def _OFFEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>;
 596     def _IDXEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>;
 597     def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>;
 598   }
 599 }
 600
 601 class MUBUF_Pseudo_Store_Lds<string opName>
 602   : MUBUF_Pseudo<opName,
 603                  (outs),
 604                  (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc, SWZ:$swz),
 605                  " $srsrc, $soffset$offset lds$glc$slc$swz"> {
 606   let mayLoad = 0;
 607   let mayStore = 1;
 608   let maybeAtomic = 1;
 609
 610   let has_vdata = 0;
 611   let has_vaddr = 0;
 612   let has_tfe = 0;
 613   let lds = 1;
 614
 615   let Uses = [EXEC, M0];
 616   let AsmMatchConverter = "cvtMubufLds";
 617 }
 618
 619 class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
 620                           list<RegisterClass> vaddrList=[]> {
 621   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 622   dag ret = !if(vdata_in,
 623     !if(!empty(vaddrList),
 624       (ins vdataClass:$vdata_in,
 625            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc),
 626       (ins vdataClass:$vdata_in, vaddrClass:$vaddr,
 627            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc)
 628     ),
 629     !if(!empty(vaddrList),
 630       (ins vdataClass:$vdata,
 631            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc),
 632       (ins vdataClass:$vdata, vaddrClass:$vaddr,
 633            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc)
 634   ));
 635 }
 636
 637 class getMUBUFAtomicIns<int addrKind,
 638                         RegisterClass vdataClass,
 639                         bit vdata_in,
 640                         // Workaround bug bz30254
 641                         RegisterClass vdataClassCopy=vdataClass> {
 642   dag ret =
 643     !if(!eq(addrKind, BUFAddrKind.Offset),
 644             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret,
 645     !if(!eq(addrKind, BUFAddrKind.OffEn),
 646             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 647     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 648             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 649     !if(!eq(addrKind, BUFAddrKind.BothEn),
 650             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 651     !if(!eq(addrKind, BUFAddrKind.Addr64),
 652             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 653     (ins))))));
 654 }
 655
 656 class MUBUF_Atomic_Pseudo<string opName,
 657                           int addrKind,
 658                           dag outs,
 659                           dag ins,
 660                           string asmOps,
 661                           list<dag> pattern=[],
 662                           // Workaround bug bz30254
 663                           int addrKindCopy = addrKind>
 664   : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,
 665     MUBUF_SetupAddr<addrKindCopy> {
 666   let mayStore = 1;
 667   let mayLoad = 1;
 668   let hasPostISelHook = 1;
 669   let hasSideEffects = 1;
 670   let DisableWQM = 1;
 671   let has_glc = 0;
 672   let has_dlc = 0;
 673   let has_tfe = 0;
 674   let maybeAtomic = 1;
 675 }
 676
 677 class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
 678                                RegisterClass vdataClass,
 679                                list<dag> pattern=[],
 680                                // Workaround bug bz30254
 681                                int addrKindCopy = addrKind,
 682                                RegisterClass vdataClassCopy = vdataClass>
 683   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 684                         (outs),
 685                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret,
 686                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc",
 687                         pattern>,
 688     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> {
 689   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 690   let glc_value = 0;
 691   let dlc_value = 0;
 692   let AsmMatchConverter = "cvtMubufAtomic";
 693 }
 694
 695 class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
 696                              RegisterClass vdataClass,
 697                              list<dag> pattern=[],
 698                              // Workaround bug bz30254
 699                              int addrKindCopy = addrKind,
 700                              RegisterClass vdataClassCopy = vdataClass>
 701   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 702                         (outs vdataClassCopy:$vdata),
 703                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret,
 704                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc",
 705                         pattern>,
 706     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> {
 707   let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret;
 708   let glc_value = 1;
 709   let dlc_value = 0;
 710   let Constraints = "$vdata = $vdata_in";
 711   let DisableEncoding = "$vdata_in";
 712   let AsmMatchConverter = "cvtMubufAtomicReturn";
 713 }
 714
 715 multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName,
 716                                         RegisterClass vdataClass,
 717                                         ValueType vdataType,
 718                                         SDPatternOperator atomic,
 719                                         bit isFP = isFloatType<vdataType>.ret> {
 720   let FPAtomic = isFP in
 721   def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>,
 722                 MUBUFAddr64Table <0, NAME>;
 723
 724   let FPAtomic = isFP in
 725   def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>,
 726                 MUBUFAddr64Table <1, NAME>;
 727
 728   let FPAtomic = isFP in
 729   def _OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 730
 731   let FPAtomic = isFP in
 732
 733   def _IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 734
 735   let FPAtomic = isFP in
 736   def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 737 }
 738
 739 multiclass MUBUF_Pseudo_Atomics_RTN <string opName,
 740                                      RegisterClass vdataClass,
 741                                      ValueType vdataType,
 742                                      SDPatternOperator atomic,
 743                                      bit isFP = isFloatType<vdataType>.ret> {
 744   let FPAtomic = isFP in
 745   def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 746     [(set vdataType:$vdata,
 747      (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc),
 748              vdataType:$vdata_in))]>,
 749     MUBUFAddr64Table <0, NAME # "_RTN">;
 750
 751   let FPAtomic = isFP in
 752   def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 753     [(set vdataType:$vdata,
 754      (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc),
 755              vdataType:$vdata_in))]>,
 756     MUBUFAddr64Table <1, NAME # "_RTN">;
 757
 758   let FPAtomic = isFP in
 759   def _OFFEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 760
 761   let FPAtomic = isFP in
 762   def _IDXEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 763
 764   let FPAtomic = isFP in
 765   def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 766 }
 767
 768 multiclass MUBUF_Pseudo_Atomics <string opName,
 769                                  RegisterClass vdataClass,
 770                                  ValueType vdataType,
 771                                  SDPatternOperator atomic> :
 772   MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>,
 773   MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>;
 774
 775
 776 //===----------------------------------------------------------------------===//
 777 // MUBUF Instructions
 778 //===----------------------------------------------------------------------===//
 779
 780 defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds <
 781   "buffer_load_format_x", f32
 782 >;
 783 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
 784   "buffer_load_format_xy", v2f32
 785 >;
 786 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <
 787   "buffer_load_format_xyz", v3f32
 788 >;
 789 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <
 790   "buffer_load_format_xyzw", v4f32
 791 >;
 792 defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <
 793   "buffer_store_format_x", f32
 794 >;
 795 defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <
 796   "buffer_store_format_xy", v2f32
 797 >;
 798 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <
 799   "buffer_store_format_xyz", v3f32
 800 >;
 801 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <
 802   "buffer_store_format_xyzw", v4f32
 803 >;
 804
 805 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
 806   defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads <
 807     "buffer_load_format_d16_x", i32
 808   >;
 809   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads <
 810     "buffer_load_format_d16_xy", v2i32
 811   >;
 812   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads <
 813     "buffer_load_format_d16_xyz", v3i32
 814   >;
 815   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads <
 816    "buffer_load_format_d16_xyzw", v4i32
 817   >;
 818   defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores <
 819     "buffer_store_format_d16_x", i32
 820   >;
 821   defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores <
 822     "buffer_store_format_d16_xy", v2i32
 823   >;
 824   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores <
 825     "buffer_store_format_d16_xyz", v3i32
 826   >;
 827   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores <
 828     "buffer_store_format_d16_xyzw", v4i32
 829   >;
 830 } // End HasUnpackedD16VMem.
 831
 832 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
 833   defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads <
 834     "buffer_load_format_d16_x", f16
 835   >;
 836   defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads <
 837     "buffer_load_format_d16_xy", v2f16
 838   >;
 839   defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads <
 840     "buffer_load_format_d16_xyz", v3f16
 841   >;
 842   defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads <
 843     "buffer_load_format_d16_xyzw", v4f16
 844   >;
 845   defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores <
 846     "buffer_store_format_d16_x", f16
 847   >;
 848   defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores <
 849     "buffer_store_format_d16_xy", v2f16
 850   >;
 851   defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores <
 852     "buffer_store_format_d16_xyz", v3f16
 853   >;
 854   defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores <
 855     "buffer_store_format_d16_xyzw", v4f16
 856   >;
 857 } // End HasPackedD16VMem.
 858
 859 defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds <
 860   "buffer_load_ubyte", i32
 861 >;
 862 defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds <
 863   "buffer_load_sbyte", i32
 864 >;
 865 defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds <
 866   "buffer_load_ushort", i32
 867 >;
 868 defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds <
 869   "buffer_load_sshort", i32
 870 >;
 871 defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds <
 872   "buffer_load_dword", i32
 873 >;
 874 defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
 875   "buffer_load_dwordx2", v2i32
 876 >;
 877 defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
 878   "buffer_load_dwordx3", v3i32
 879 >;
 880 defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
 881   "buffer_load_dwordx4", v4i32
 882 >;
 883
 884 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>;
 885 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>;
 886 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>;
 887 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>;
 888 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>;
 889 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>;
 890 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>;
 891 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>;
 892 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>;
 893 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>;
 894
 895 // This is not described in AMD documentation,
 896 // but 'lds' versions of these opcodes are available
 897 // in at least GFX8+ chips. See Bug 37653.
 898 let SubtargetPredicate = isGFX8GFX9 in {
 899 defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads <
 900   "buffer_load_dwordx2", v2i32, null_frag, 0, 1
 901 >;
 902 defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads <
 903   "buffer_load_dwordx3", v3i32, null_frag, 0, 1
 904 >;
 905 defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads <
 906   "buffer_load_dwordx4", v4i32, null_frag, 0, 1
 907 >;
 908 }
 909
 910 defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
 911   "buffer_store_byte", i32, truncstorei8_global
 912 >;
 913 defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
 914   "buffer_store_short", i32, truncstorei16_global
 915 >;
 916 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
 917   "buffer_store_dword", i32, store_global
 918 >;
 919 defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
 920   "buffer_store_dwordx2", v2i32, store_global
 921 >;
 922 defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
 923   "buffer_store_dwordx3", v3i32, store_global
 924 >;
 925 defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
 926   "buffer_store_dwordx4", v4i32, store_global
 927 >;
 928 defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
 929   "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32
 930 >;
 931 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <
 932   "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag
 933 >;
 934 defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <
 935   "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32
 936 >;
 937 defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <
 938   "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32
 939 >;
 940 defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <
 941   "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32
 942 >;
 943 defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <
 944   "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32
 945 >;
 946 defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <
 947   "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32
 948 >;
 949 defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <
 950   "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32
 951 >;
 952 defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <
 953   "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32
 954 >;
 955 defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <
 956   "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32
 957 >;
 958 defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <
 959   "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32
 960 >;
 961 defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <
 962   "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32
 963 >;
 964 defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <
 965   "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32
 966 >;
 967 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <
 968   "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64
 969 >;
 970 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <
 971   "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag
 972 >;
 973 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <
 974   "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64
 975 >;
 976 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <
 977   "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64
 978 >;
 979 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <
 980   "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64
 981 >;
 982 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <
 983   "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64
 984 >;
 985 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <
 986   "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64
 987 >;
 988 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <
 989   "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64
 990 >;
 991 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <
 992   "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64
 993 >;
 994 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <
 995   "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64
 996 >;
 997 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <
 998   "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64
 999 >;
1000 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <
1001   "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64
1002 >;
1003 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
1004   "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64
1005 >;
1006
1007 let SubtargetPredicate = HasGFX10_BEncoding in
1008 defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN <
1009   "buffer_atomic_csub", VGPR_32, i32, atomic_csub_global_32
1010 >;
1011
1012 let SubtargetPredicate = isGFX8GFX9 in {
1013 def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">;
1014 }
1015
1016 let SubtargetPredicate = isGFX6 in { // isn't on CI & VI
1017 /*
1018 defm BUFFER_ATOMIC_RSUB        : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
1019 defm BUFFER_ATOMIC_RSUB_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
1020 */
1021
1022 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
1023                                           int_amdgcn_buffer_wbinvl1_sc>;
1024 }
1025
1026 let SubtargetPredicate = isGFX6GFX7GFX10 in {
1027
1028 defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <
1029   "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag
1030 >;
1031 defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <
1032   "buffer_atomic_fmin", VGPR_32, f32, null_frag
1033 >;
1034 defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <
1035   "buffer_atomic_fmax", VGPR_32, f32, null_frag
1036 >;
1037 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <
1038   "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag
1039 >;
1040 defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <
1041   "buffer_atomic_fmin_x2", VReg_64, f64, null_frag
1042 >;
1043 defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <
1044   "buffer_atomic_fmax_x2", VReg_64, f64, null_frag
1045 >;
1046
1047 }
1048
1049 let SubtargetPredicate = HasD16LoadStore in {
1050
1051 defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads <
1052   "buffer_load_ubyte_d16", i32, null_frag, 1
1053 >;
1054
1055 defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads <
1056   "buffer_load_ubyte_d16_hi", i32, null_frag, 1
1057 >;
1058
1059 defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads <
1060   "buffer_load_sbyte_d16", i32, null_frag, 1
1061 >;
1062
1063 defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads <
1064   "buffer_load_sbyte_d16_hi", i32, null_frag, 1
1065 >;
1066
1067 defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads <
1068   "buffer_load_short_d16", i32, null_frag, 1
1069 >;
1070
1071 defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads <
1072   "buffer_load_short_d16_hi", i32, null_frag, 1
1073 >;
1074
1075 defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores <
1076   "buffer_store_byte_d16_hi", i32
1077 >;
1078
1079 defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores <
1080   "buffer_store_short_d16_hi", i32
1081 >;
1082
1083 defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads <
1084   "buffer_load_format_d16_hi_x", i32
1085 >;
1086 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores <
1087   "buffer_store_format_d16_hi_x", i32
1088 >;
1089
1090 } // End HasD16LoadStore
1091
1092 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
1093                                        int_amdgcn_buffer_wbinvl1>;
1094
1095 let SubtargetPredicate = HasAtomicFaddInsts in {
1096
1097 defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN <
1098   "buffer_atomic_add_f32", VGPR_32, f32, atomic_fadd_global_noret
1099 >;
1100 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN <
1101   "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_pk_fadd_global_noret
1102 >;
1103
1104 } // End SubtargetPredicate = HasAtomicFaddInsts
1105
1106 //===----------------------------------------------------------------------===//
1107 // MTBUF Instructions
1108 //===----------------------------------------------------------------------===//
1109
1110 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32,  1>;
1111 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xy",    VReg_64,  2>;
1112 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyz",   VReg_96,  3>;
1113 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyzw",  VReg_128, 4>;
1114 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_x",    VGPR_32,  1>;
1115 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy",   VReg_64,  2>;
1116 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz",  VReg_96,  3>;
1117 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>;
1118
1119 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
1120   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32,  1>;
1121   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VReg_64,  2>;
1122   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_96,  3>;
1123   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_128, 4>;
1124   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32,  1>;
1125   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VReg_64,  2>;
1126   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_96,  3>;
1127   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>;
1128 } // End HasUnpackedD16VMem.
1129
1130 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
1131   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32, 1>;
1132   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VGPR_32, 2>;
1133   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_64, 3>;
1134   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_64, 4>;
1135   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32, 1>;
1136   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VGPR_32, 2>;
1137   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_64, 3>;
1138   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>;
1139 } // End HasPackedD16VMem.
1140
1141 let SubtargetPredicate = isGFX7Plus in {
1142
1143 //===----------------------------------------------------------------------===//
1144 // Instruction definitions for CI and newer.
1145 //===----------------------------------------------------------------------===//
1146
1147 def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
1148                                            int_amdgcn_buffer_wbinvl1_vol>;
1149
1150 } // End let SubtargetPredicate = isGFX7Plus
1151
1152 let SubtargetPredicate = isGFX10Plus in {
1153   def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">;
1154   def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">;
1155 } // End SubtargetPredicate = isGFX10Plus
1156
1157 //===----------------------------------------------------------------------===//
1158 // MUBUF Patterns
1159 //===----------------------------------------------------------------------===//
1160
1161 //===----------------------------------------------------------------------===//
1162 // buffer_load/store_format patterns
1163 //===----------------------------------------------------------------------===//
1164
1165 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1166                                   string opcode> {
1167   def : GCNPat<
1168     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1169               timm:$auxiliary, 0)),
1170     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1171       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1172       (extract_swz $auxiliary))
1173   >;
1174
1175   def : GCNPat<
1176     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1177               timm:$auxiliary, 0)),
1178     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1179       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1180       (extract_swz $auxiliary))
1181   >;
1182
1183   def : GCNPat<
1184     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1185               timm:$auxiliary, timm)),
1186     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1187       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1188       (extract_swz $auxiliary))
1189   >;
1190
1191   def : GCNPat<
1192     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1193               timm:$auxiliary, timm)),
1194     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1195       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
1196       SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1197       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1198       (extract_swz $auxiliary))
1199   >;
1200 }
1201
1202 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
1203 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">;
1204 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
1205 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">;
1206 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">;
1207 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">;
1208 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;
1209 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">;
1210
1211 let SubtargetPredicate = HasUnpackedD16VMem in {
1212   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1213   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1214   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1215   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1216   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1217 } // End HasUnpackedD16VMem.
1218
1219 let SubtargetPredicate = HasPackedD16VMem in {
1220   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">;
1221   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">;
1222   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">;
1223   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">;
1224   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">;
1225   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1226   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1227 } // End HasPackedD16VMem.
1228
1229 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">;
1230 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">;
1231 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">;
1232 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">;
1233 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">;
1234 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">;
1235 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">;
1236 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">;
1237 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">;
1238 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">;
1239 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">;
1240 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">;
1241 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">;
1242 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">;
1243 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">;
1244 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort,  i32, "BUFFER_LOAD_USHORT">;
1245
1246 multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1247                                    string opcode> {
1248   def : GCNPat<
1249     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1250               timm:$auxiliary, 0),
1251     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1252       (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1253       (extract_swz $auxiliary))
1254   >;
1255
1256   def : GCNPat<
1257     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1258               timm:$auxiliary, 0),
1259     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset,
1260       (as_i16timm $offset), (extract_glc $auxiliary),
1261       (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1262       (extract_swz $auxiliary))
1263   >;
1264
1265   def : GCNPat<
1266     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1267               timm:$auxiliary, timm),
1268     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset,
1269       (as_i16timm $offset), (extract_glc $auxiliary),
1270       (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1271       (extract_swz $auxiliary))
1272   >;
1273
1274   def : GCNPat<
1275     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1276               timm:$auxiliary, timm),
1277     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
1278       getVregSrcForVT<vt>.ret:$vdata,
1279       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
1280       SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_glc $auxiliary),
1281       (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1282       (extract_swz $auxiliary))
1283   >;
1284 }
1285
1286 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
1287 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">;
1288 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
1289 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">;
1290 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">;
1291 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">;
1292 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
1293 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">;
1294
1295 let SubtargetPredicate = HasUnpackedD16VMem in {
1296   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1297   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1298   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1299   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">;
1300   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1301 } // End HasUnpackedD16VMem.
1302
1303 let SubtargetPredicate = HasPackedD16VMem in {
1304   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">;
1305   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">;
1306   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">;
1307   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">;
1308   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">;
1309   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1310   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1311 } // End HasPackedD16VMem.
1312
1313 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">;
1314 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">;
1315 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">;
1316 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">;
1317 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">;
1318 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">;
1319 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">;
1320 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">;
1321 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">;
1322 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">;
1323 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">;
1324 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">;
1325 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">;
1326 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">;
1327
1328 //===----------------------------------------------------------------------===//
1329 // buffer_atomic patterns
1330 //===----------------------------------------------------------------------===//
1331
1332 multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt,
1333                                 string opcode> {
1334   def : GCNPat<
1335     (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset,
1336               timm:$offset, timm:$cachepolicy, 0)),
1337     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN)
1338       getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset,
1339       (as_i16timm $offset), (extract_slc $cachepolicy))
1340   >;
1341
1342   def : GCNPat<
1343     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset,
1344               timm:$offset, timm:$cachepolicy, timm)),
1345     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in,
1346       VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset,
1347       (as_i16timm $offset), (extract_slc $cachepolicy))
1348   >;
1349
1350   def : GCNPat<
1351     (vt (name vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset,
1352               i32:$soffset, timm:$offset, timm:$cachepolicy, 0)),
1353     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in,
1354       VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset,
1355       (as_i16timm $offset), (extract_slc $cachepolicy))
1356   >;
1357
1358   def : GCNPat<
1359     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset,
1360               i32:$soffset, timm:$offset, timm:$cachepolicy, timm)),
1361     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN)
1362       getVregSrcForVT<vt>.ret:$vdata_in,
1363       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
1364         SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1365         (extract_slc $cachepolicy))
1366   >;
1367 }
1368
1369 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">;
1370 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">;
1371 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">;
1372 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">;
1373 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">;
1374 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">;
1375 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">;
1376 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">;
1377 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">;
1378 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">;
1379 defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">;
1380 defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">;
1381 defm : BufferAtomicPatterns<SIbuffer_atomic_csub, i32, "BUFFER_ATOMIC_CSUB">;
1382 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">;
1383 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64,  "BUFFER_ATOMIC_ADD_X2">;
1384 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">;
1385 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">;
1386 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">;
1387 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">;
1388 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">;
1389 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">;
1390 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">;
1391 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">;
1392 defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">;
1393 defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">;
1394
1395 multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt,
1396                                        string opcode> {
1397   def : GCNPat<
1398     (name vt:$vdata_in, v4i32:$rsrc, 0,
1399           0, i32:$soffset, timm:$offset,
1400           timm:$cachepolicy, 0),
1401     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $vdata_in, $rsrc, $soffset,
1402                                         (as_i16imm $offset), (extract_slc $cachepolicy))
1403   >;
1404
1405   def : GCNPat<
1406     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1407           0, i32:$soffset, timm:$offset,
1408           timm:$cachepolicy, timm),
1409     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vdata_in, $vindex, $rsrc, $soffset,
1410                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1411   >;
1412
1413   def : GCNPat<
1414     (name vt:$vdata_in, v4i32:$rsrc, 0,
1415           i32:$voffset, i32:$soffset, timm:$offset,
1416           timm:$cachepolicy, 0),
1417     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $vdata_in, $voffset, $rsrc, $soffset,
1418                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1419   >;
1420
1421   def : GCNPat<
1422     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1423           i32:$voffset, i32:$soffset, timm:$offset,
1424           timm:$cachepolicy, timm),
1425     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1426       $vdata_in,
1427       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1428       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy))
1429   >;
1430 }
1431
1432 let SubtargetPredicate = HasAtomicFaddInsts in {
1433 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">;
1434 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_pk_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">;
1435 }
1436
1437 def : GCNPat<
1438   (SIbuffer_atomic_cmpswap
1439       i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset,
1440       timm:$offset, timm:$cachepolicy, 0),
1441   (EXTRACT_SUBREG
1442     (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN
1443       (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1),
1444         SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1445         (extract_slc $cachepolicy)), sub0)
1446 >;
1447
1448 def : GCNPat<
1449   (SIbuffer_atomic_cmpswap
1450       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1451       0, i32:$soffset, timm:$offset,
1452       timm:$cachepolicy, timm),
1453   (EXTRACT_SUBREG
1454     (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN
1455       (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1),
1456       VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)),
1457     sub0)
1458 >;
1459
1460 def : GCNPat<
1461   (SIbuffer_atomic_cmpswap
1462       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1463       i32:$voffset, i32:$soffset, timm:$offset,
1464       timm:$cachepolicy, 0),
1465   (EXTRACT_SUBREG
1466     (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN
1467       (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1),
1468       VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)),
1469     sub0)
1470 >;
1471
1472 def : GCNPat<
1473   (SIbuffer_atomic_cmpswap
1474       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1475       i32:$voffset, i32:$soffset, timm:$offset,
1476       timm:$cachepolicy, timm),
1477   (EXTRACT_SUBREG
1478     (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN
1479       (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1),
1480       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
1481       SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)),
1482     sub0)
1483 >;
1484
1485 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
1486                               PatFrag constant_ld> : GCNPat <
1487      (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1488                                    i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
1489      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz)
1490   >;
1491
1492 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1493                                      ValueType vt, PatFrag atomic_ld> {
1494   def : GCNPat <
1495      (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1496                                    i16:$offset, i1:$slc))),
1497      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0)
1498   >;
1499
1500   def : GCNPat <
1501     (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))),
1502     (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0)
1503   >;
1504 }
1505
1506 let SubtargetPredicate = isGFX6GFX7 in {
1507 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
1508 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>;
1509 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>;
1510 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
1511 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>;
1512 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>;
1513
1514 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>;
1515 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>;
1516 } // End SubtargetPredicate = isGFX6GFX7
1517
1518 multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1519                                PatFrag ld> {
1520
1521   def : GCNPat <
1522     (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1523                           i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
1524     (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz)
1525   >;
1526 }
1527
1528 let OtherPredicates = [Has16BitInsts] in {
1529
1530 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>;
1531 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>;
1532 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>;
1533 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>;
1534 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>;
1535 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>;
1536
1537 defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>;
1538
1539 } // End OtherPredicates = [Has16BitInsts]
1540
1541 multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen,
1542                                 MUBUF_Pseudo InstrOffset,
1543                                 ValueType vt, PatFrag ld> {
1544   def : GCNPat <
1545     (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1546                                i32:$soffset, u16imm:$offset))),
1547     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
1548   >;
1549
1550   def : GCNPat <
1551     (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))),
1552     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
1553   >;
1554 }
1555
1556 // XXX - Is it possible to have a complex pattern in a PatFrag?
1557 multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen,
1558                                 MUBUF_Pseudo InstrOffset,
1559                                 ValueType vt, PatFrag ld_frag> {
1560   def : GCNPat <
1561     (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in),
1562     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in)
1563   >;
1564
1565   def : GCNPat <
1566     (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in),
1567     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in)
1568   >;
1569 }
1570
1571 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>;
1572 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>;
1573 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>;
1574 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>;
1575 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>;
1576 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>;
1577 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>;
1578 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>;
1579 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>;
1580 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>;
1581
1582 foreach vt = Reg32Types.types in {
1583 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>;
1584 }
1585 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>;
1586 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>;
1587 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>;
1588
1589 let OtherPredicates = [D16PreservesUnusedBits] in {
1590 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>;
1591 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>;
1592 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>;
1593 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>;
1594 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>;
1595 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>;
1596
1597 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>;
1598 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>;
1599 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>;
1600 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>;
1601 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>;
1602 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>;
1603 }
1604
1605 multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1606                                       ValueType vt, PatFrag atomic_st> {
1607   // Store follows atomic op convention so address is first
1608   def : GCNPat <
1609      (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1610                                    i16:$offset, i1:$slc), vt:$val),
1611      (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0)
1612   >;
1613
1614   def : GCNPat <
1615     (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
1616     (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0)
1617   >;
1618 }
1619 let SubtargetPredicate = isGFX6GFX7 in {
1620 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_global_32>;
1621 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_global_64>;
1622 } // End Predicates = isGFX6GFX7
1623
1624
1625 multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1626                                PatFrag st> {
1627
1628   def : GCNPat <
1629     (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1630                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)),
1631     (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz)
1632   >;
1633 }
1634
1635 defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>;
1636 defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>;
1637
1638 multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen,
1639                                  MUBUF_Pseudo InstrOffset,
1640                                  ValueType vt, PatFrag st,
1641                                  RegisterClass rc = VGPR_32> {
1642   def : GCNPat <
1643     (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1644                                       i32:$soffset, u16imm:$offset)),
1645     (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
1646   >;
1647
1648   def : GCNPat <
1649     (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,
1650                                        u16imm:$offset)),
1651     (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
1652   >;
1653 }
1654
1655 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>;
1656 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>;
1657 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>;
1658 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>;
1659
1660 foreach vt = Reg32Types.types in {
1661 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>;
1662 }
1663
1664 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>;
1665 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>;
1666 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>;
1667
1668
1669 let OtherPredicates = [D16PreservesUnusedBits] in {
1670  // Hiding the extract high pattern in the PatFrag seems to not
1671  // automatically increase the complexity.
1672 let AddedComplexity = 1 in {
1673 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>;
1674 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>;
1675 }
1676 }
1677
1678 //===----------------------------------------------------------------------===//
1679 // MTBUF Patterns
1680 //===----------------------------------------------------------------------===//
1681
1682 //===----------------------------------------------------------------------===//
1683 // tbuffer_load/store_format patterns
1684 //===----------------------------------------------------------------------===//
1685
1686 multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1687                                   string opcode> {
1688   def : GCNPat<
1689     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1690               timm:$format, timm:$auxiliary, 0)),
1691     (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1692       (as_i8timm $format),
1693       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1694       (extract_swz $auxiliary))
1695   >;
1696
1697   def : GCNPat<
1698     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1699               timm:$format, timm:$auxiliary, timm)),
1700     (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1701       (as_i8timm $format),
1702       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1703       (extract_swz $auxiliary))
1704   >;
1705
1706   def : GCNPat<
1707     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1708               timm:$format, timm:$auxiliary, 0)),
1709     (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1710       (as_i8timm $format),
1711       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1712       (extract_swz $auxiliary))
1713   >;
1714
1715   def : GCNPat<
1716     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1717               timm:$format, timm:$auxiliary, timm)),
1718     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN)
1719       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
1720       SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1721       (as_i8timm $format),
1722       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1723       (extract_swz $auxiliary))
1724   >;
1725 }
1726
1727 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32,   "TBUFFER_LOAD_FORMAT_X">;
1728 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;
1729 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">;
1730 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">;
1731 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32,   "TBUFFER_LOAD_FORMAT_X">;
1732 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;
1733 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">;
1734 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;
1735
1736 let SubtargetPredicate = HasUnpackedD16VMem in {
1737   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
1738   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32,   "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
1739   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1740   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1741 } // End HasUnpackedD16VMem.
1742
1743 let SubtargetPredicate = HasPackedD16VMem in {
1744   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X">;
1745   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32,   "TBUFFER_LOAD_FORMAT_D16_X">;
1746   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">;
1747   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
1748 } // End HasPackedD16VMem.
1749
1750 multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1751                                    string opcode> {
1752   def : GCNPat<
1753     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1754           timm:$format, timm:$auxiliary, 0),
1755     (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset,
1756       (as_i16timm $offset), (as_i8timm $format),
1757       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1758       (extract_swz $auxiliary))
1759   >;
1760
1761   def : GCNPat<
1762     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1763           timm:$format, timm:$auxiliary, timm),
1764     (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset,
1765       (as_i16timm $offset), (as_i8timm $format),
1766       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1767       (extract_swz $auxiliary))
1768   >;
1769
1770   def : GCNPat<
1771     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1772           timm:$format, timm:$auxiliary, 0),
1773     (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset,
1774       (as_i16timm $offset), (as_i8timm $format),
1775       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1776       (extract_swz $auxiliary))
1777   >;
1778
1779   def : GCNPat<
1780     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset,
1781           timm:$offset, timm:$format, timm:$auxiliary, timm),
1782     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact)
1783       getVregSrcForVT<vt>.ret:$vdata,
1784       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
1785       SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format),
1786       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1787       (extract_swz $auxiliary))
1788   >;
1789 }
1790
1791 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32,   "TBUFFER_STORE_FORMAT_X">;
1792 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;
1793 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">;
1794 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">;
1795 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32,   "TBUFFER_STORE_FORMAT_X">;
1796 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;
1797 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">;
1798 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;
1799
1800 let SubtargetPredicate = HasUnpackedD16VMem in {
1801   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
1802   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32,   "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
1803   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">;
1804   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1805 } // End HasUnpackedD16VMem.
1806
1807 let SubtargetPredicate = HasPackedD16VMem in {
1808   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X">;
1809   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32,   "TBUFFER_STORE_FORMAT_D16_X">;
1810   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">;
1811   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
1812 } // End HasPackedD16VMem.
1813
1814 //===----------------------------------------------------------------------===//
1815 // Target-specific instruction encodings.
1816 //===----------------------------------------------------------------------===//
1817
1818 //===----------------------------------------------------------------------===//
1819 // Base ENC_MUBUF for GFX6, GFX7, GFX10.
1820 //===----------------------------------------------------------------------===//
1821
1822 class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> :
1823     MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
1824   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1825   let Inst{12}    = ps.offen;
1826   let Inst{13}    = ps.idxen;
1827   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1828   let Inst{16}    = !if(ps.lds, 1, 0);
1829   let Inst{24-18} = op;
1830   let Inst{31-26} = 0x38;
1831   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1832   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1833   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1834   let Inst{54}    = !if(ps.has_slc, slc, ?);
1835   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1836   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1837 }
1838
1839 class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> :
1840     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> {
1841   let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value);
1842   let Inst{25} = op{7};
1843 }
1844
1845 class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> :
1846     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> {
1847   let Inst{15} = ps.addr64;
1848 }
1849
1850 //===----------------------------------------------------------------------===//
1851 // MUBUF - GFX10.
1852 //===----------------------------------------------------------------------===//
1853
1854 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
1855   multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName,
1856                                         string asmName> {
1857     def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> {
1858       MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName);
1859       let AsmString = asmName # ps.AsmOperands;
1860     }
1861   }
1862   multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> {
1863     def _BOTHEN_gfx10 :
1864       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1865     def _IDXEN_gfx10 :
1866       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1867     def _OFFEN_gfx10 :
1868       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1869     def _OFFSET_gfx10 :
1870       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1871   }
1872   multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> {
1873     def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1874                         MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">;
1875     def _OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1876                         MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">;
1877     def _IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1878                         MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">;
1879     def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1880                         MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">;
1881
1882     def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
1883                             MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">;
1884     def _LDS_OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
1885                             MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">;
1886     def _LDS_IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
1887                             MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">;
1888     def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
1889                             MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">;
1890   }
1891   multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> {
1892     def _BOTHEN_RTN_gfx10 :
1893       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1894     def _IDXEN_RTN_gfx10 :
1895       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1896     def _OFFEN_RTN_gfx10 :
1897       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1898     def _OFFSET_RTN_gfx10 :
1899       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1900   }
1901   multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> :
1902       MUBUF_Real_AllAddr_gfx10<op>, MUBUF_Real_Atomics_RTN_gfx10<op>;
1903 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
1904
1905 defm BUFFER_STORE_BYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x019>;
1906 defm BUFFER_STORE_SHORT_D16_HI    : MUBUF_Real_AllAddr_gfx10<0x01b>;
1907 defm BUFFER_LOAD_UBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x020>;
1908 defm BUFFER_LOAD_UBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x021>;
1909 defm BUFFER_LOAD_SBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x022>;
1910 defm BUFFER_LOAD_SBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x023>;
1911 defm BUFFER_LOAD_SHORT_D16        : MUBUF_Real_AllAddr_gfx10<0x024>;
1912 defm BUFFER_LOAD_SHORT_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x025>;
1913 // FIXME-GFX10: Add following instructions:
1914 //defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_gfx10<0x026>;
1915 //defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>;
1916 defm BUFFER_LOAD_FORMAT_D16_X     : MUBUF_Real_AllAddr_gfx10<0x080>;
1917 defm BUFFER_LOAD_FORMAT_D16_XY    : MUBUF_Real_AllAddr_gfx10<0x081>;
1918 defm BUFFER_LOAD_FORMAT_D16_XYZ   : MUBUF_Real_AllAddr_gfx10<0x082>;
1919 defm BUFFER_LOAD_FORMAT_D16_XYZW  : MUBUF_Real_AllAddr_gfx10<0x083>;
1920 defm BUFFER_STORE_FORMAT_D16_X    : MUBUF_Real_AllAddr_gfx10<0x084>;
1921 defm BUFFER_STORE_FORMAT_D16_XY   : MUBUF_Real_AllAddr_gfx10<0x085>;
1922 defm BUFFER_STORE_FORMAT_D16_XYZ  : MUBUF_Real_AllAddr_gfx10<0x086>;
1923 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>;
1924
1925 def BUFFER_GL0_INV_gfx10 :
1926   MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>;
1927 def BUFFER_GL1_INV_gfx10 :
1928   MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>;
1929
1930 //===----------------------------------------------------------------------===//
1931 // MUBUF - GFX6, GFX7, GFX10.
1932 //===----------------------------------------------------------------------===//
1933
1934 let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in {
1935   multiclass MUBUF_Real_gfx6<bits<8> op> {
1936     def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
1937   }
1938 } // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6"
1939
1940 let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {
1941   multiclass MUBUF_Real_gfx7<bits<8> op> {
1942     def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
1943   }
1944 } // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"
1945
1946 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
1947   multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> {
1948     def _ADDR64_gfx6_gfx7 :
1949       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>;
1950     def _BOTHEN_gfx6_gfx7 :
1951       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1952     def _IDXEN_gfx6_gfx7 :
1953       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1954     def _OFFEN_gfx6_gfx7 :
1955       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1956     def _OFFSET_gfx6_gfx7 :
1957       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1958   }
1959   multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> {
1960     def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1961                             MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">;
1962     def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>,
1963                             MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">;
1964     def _OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1965                             MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">;
1966     def _IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1967                             MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">;
1968     def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1969                             MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">;
1970
1971     def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
1972                                 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">;
1973     def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>,
1974                                 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">;
1975     def _LDS_OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
1976                                 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">;
1977     def _LDS_IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
1978                                 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">;
1979     def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
1980                                 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">;
1981   }
1982   multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> :
1983       MUBUF_Real_AllAddr_gfx6_gfx7<op> {
1984     def _ADDR64_RTN_gfx6_gfx7 :
1985       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>;
1986     def _BOTHEN_RTN_gfx6_gfx7 :
1987       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1988     def _IDXEN_RTN_gfx6_gfx7 :
1989       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1990     def _OFFEN_RTN_gfx6_gfx7 :
1991       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1992     def _OFFSET_RTN_gfx6_gfx7 :
1993       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1994   }
1995 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
1996
1997 multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> :
1998   MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>;
1999
2000 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> :
2001   MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>;
2002
2003 multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> :
2004   MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>;
2005
2006 // FIXME-GFX6: Following instructions are available only on GFX6.
2007 //defm BUFFER_ATOMIC_RSUB         : MUBUF_Real_Atomics_gfx6 <0x034>;
2008 //defm BUFFER_ATOMIC_RSUB_X2      : MUBUF_Real_Atomics_gfx6 <0x054>;
2009
2010 defm BUFFER_LOAD_FORMAT_X     : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>;
2011 defm BUFFER_LOAD_FORMAT_XY    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
2012 defm BUFFER_LOAD_FORMAT_XYZ   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
2013 defm BUFFER_LOAD_FORMAT_XYZW  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
2014 defm BUFFER_STORE_FORMAT_X    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
2015 defm BUFFER_STORE_FORMAT_XY   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
2016 defm BUFFER_STORE_FORMAT_XYZ  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
2017 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
2018 defm BUFFER_LOAD_UBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>;
2019 defm BUFFER_LOAD_SBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>;
2020 defm BUFFER_LOAD_USHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>;
2021 defm BUFFER_LOAD_SSHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>;
2022 defm BUFFER_LOAD_DWORD        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>;
2023 defm BUFFER_LOAD_DWORDX2      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>;
2024 defm BUFFER_LOAD_DWORDX4      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>;
2025 defm BUFFER_LOAD_DWORDX3      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>;
2026 defm BUFFER_STORE_BYTE        : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>;
2027 defm BUFFER_STORE_SHORT       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>;
2028 defm BUFFER_STORE_DWORD       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>;
2029 defm BUFFER_STORE_DWORDX2     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>;
2030 defm BUFFER_STORE_DWORDX4     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>;
2031 defm BUFFER_STORE_DWORDX3     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>;
2032
2033 defm BUFFER_ATOMIC_SWAP        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>;
2034 defm BUFFER_ATOMIC_CMPSWAP     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>;
2035 defm BUFFER_ATOMIC_ADD         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>;
2036 defm BUFFER_ATOMIC_SUB         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>;
2037 defm BUFFER_ATOMIC_SMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>;
2038 defm BUFFER_ATOMIC_UMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>;
2039 defm BUFFER_ATOMIC_SMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>;
2040 defm BUFFER_ATOMIC_UMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>;
2041 defm BUFFER_ATOMIC_AND         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>;
2042 defm BUFFER_ATOMIC_OR          : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>;
2043 defm BUFFER_ATOMIC_XOR         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>;
2044 defm BUFFER_ATOMIC_INC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>;
2045 defm BUFFER_ATOMIC_DEC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>;
2046 defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>;
2047 defm BUFFER_ATOMIC_FMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>;
2048 defm BUFFER_ATOMIC_FMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>;
2049 defm BUFFER_ATOMIC_SWAP_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>;
2050 defm BUFFER_ATOMIC_CMPSWAP_X2  : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>;
2051 defm BUFFER_ATOMIC_ADD_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>;
2052 defm BUFFER_ATOMIC_SUB_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>;
2053 defm BUFFER_ATOMIC_SMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>;
2054 defm BUFFER_ATOMIC_UMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>;
2055 defm BUFFER_ATOMIC_SMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>;
2056 defm BUFFER_ATOMIC_UMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>;
2057 defm BUFFER_ATOMIC_AND_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>;
2058 defm BUFFER_ATOMIC_OR_X2       : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>;
2059 defm BUFFER_ATOMIC_XOR_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>;
2060 defm BUFFER_ATOMIC_INC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>;
2061 defm BUFFER_ATOMIC_DEC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>;
2062 // FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7.
2063 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>;
2064 defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>;
2065 defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>;
2066
2067 defm BUFFER_ATOMIC_CSUB       : MUBUF_Real_Atomics_RTN_gfx10<0x034>;
2068
2069 defm BUFFER_WBINVL1_SC        : MUBUF_Real_gfx6<0x070>;
2070 defm BUFFER_WBINVL1_VOL       : MUBUF_Real_gfx7<0x070>;
2071 def  BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>;
2072
2073 //===----------------------------------------------------------------------===//
2074 // Base ENC_MTBUF for GFX6, GFX7, GFX10.
2075 //===----------------------------------------------------------------------===//
2076
2077 class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> :
2078     MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
2079   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2080   let Inst{12}    = ps.offen;
2081   let Inst{13}    = ps.idxen;
2082   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2083   let Inst{18-16} = op;
2084   let Inst{31-26} = 0x3a; //encoding
2085   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2086   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2087   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2088   let Inst{54}    = !if(ps.has_slc, slc, ?);
2089   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2090   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2091 }
2092
2093 //===----------------------------------------------------------------------===//
2094 // MTBUF - GFX10.
2095 //===----------------------------------------------------------------------===//
2096
2097 class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> :
2098     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> {
2099   let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value);
2100   let Inst{25-19} = format;
2101   let Inst{53} = op{3};
2102 }
2103
2104 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
2105   multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> {
2106     def _BOTHEN_gfx10 :
2107       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2108     def _IDXEN_gfx10 :
2109       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2110     def _OFFEN_gfx10 :
2111       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2112     def _OFFSET_gfx10 :
2113       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2114   }
2115 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
2116
2117 defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_gfx10<0x008>;
2118 defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_gfx10<0x009>;
2119 defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_gfx10<0x00a>;
2120 defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_gfx10<0x00b>;
2121 defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_gfx10<0x00c>;
2122 defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_gfx10<0x00d>;
2123 defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_gfx10<0x00e>;
2124 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>;
2125
2126 //===----------------------------------------------------------------------===//
2127 // MTBUF - GFX6, GFX7, GFX10.
2128 //===----------------------------------------------------------------------===//
2129
2130 class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> :
2131     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> {
2132   let Inst{15} = ps.addr64;
2133   let Inst{22-19} = dfmt;
2134   let Inst{25-23} = nfmt;
2135 }
2136
2137 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
2138   multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> {
2139     def _ADDR64_gfx6_gfx7 :
2140       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>;
2141     def _BOTHEN_gfx6_gfx7 :
2142       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2143     def _IDXEN_gfx6_gfx7 :
2144       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2145     def _OFFEN_gfx6_gfx7 :
2146       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2147     def _OFFSET_gfx6_gfx7 :
2148       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2149   }
2150 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
2151
2152 multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> :
2153   MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>;
2154
2155 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>;
2156 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
2157 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
2158 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
2159 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
2160 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
2161 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
2162 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
2163
2164 //===----------------------------------------------------------------------===//
2165 // GFX8, GFX9 (VI).
2166 //===----------------------------------------------------------------------===//
2167
2168 class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> :
2169   MUBUF_Real<ps>,
2170   Enc64,
2171   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
2172   let AssemblerPredicate = isGFX8GFX9;
2173   let DecoderNamespace = "GFX8";
2174
2175   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2176   let Inst{12}    = ps.offen;
2177   let Inst{13}    = ps.idxen;
2178   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2179   let Inst{16}    = !if(ps.lds, 1, 0);
2180   let Inst{17}    = !if(ps.has_slc, slc, ?);
2181   let Inst{24-18} = op;
2182   let Inst{31-26} = 0x38; //encoding
2183   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2184   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2185   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2186   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2187   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2188 }
2189
2190 multiclass MUBUF_Real_AllAddr_vi<bits<7> op> {
2191   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2192   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2193   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2194   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2195 }
2196
2197 multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> {
2198
2199   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
2200                    MUBUFLdsTable<0, NAME # "_OFFSET_vi">;
2201   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
2202                    MUBUFLdsTable<0, NAME # "_OFFEN_vi">;
2203   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
2204                    MUBUFLdsTable<0, NAME # "_IDXEN_vi">;
2205   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
2206                    MUBUFLdsTable<0, NAME # "_BOTHEN_vi">;
2207
2208   def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
2209                        MUBUFLdsTable<1, NAME # "_OFFSET_vi">;
2210   def _LDS_OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
2211                        MUBUFLdsTable<1, NAME # "_OFFEN_vi">;
2212   def _LDS_IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
2213                        MUBUFLdsTable<1, NAME # "_IDXEN_vi">;
2214   def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
2215                        MUBUFLdsTable<1, NAME # "_BOTHEN_vi">;
2216 }
2217
2218 class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> :
2219   MUBUF_Real<ps>,
2220   Enc64,
2221   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2222   let AssemblerPredicate=HasUnpackedD16VMem;
2223   let DecoderNamespace="GFX80_UNPACKED";
2224
2225   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2226   let Inst{12}    = ps.offen;
2227   let Inst{13}    = ps.idxen;
2228   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2229   let Inst{16}    = !if(ps.lds, 1, 0);
2230   let Inst{17}    = !if(ps.has_slc, slc, ?);
2231   let Inst{24-18} = op;
2232   let Inst{31-26} = 0x38; //encoding
2233   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2234   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2235   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2236   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2237   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2238 }
2239
2240 multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> {
2241   def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2242   def _OFFEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2243   def _IDXEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2244   def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2245 }
2246
2247 multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
2248   MUBUF_Real_AllAddr_vi<op> {
2249   def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
2250   def _OFFEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
2251   def _IDXEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
2252   def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
2253 }
2254
2255 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_Lds_vi <0x00>;
2256 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_vi <0x01>;
2257 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_vi <0x02>;
2258 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_vi <0x03>;
2259 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_vi <0x04>;
2260 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_vi <0x05>;
2261 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_vi <0x06>;
2262 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_vi <0x07>;
2263 let SubtargetPredicate = HasUnpackedD16VMem in {
2264   defm BUFFER_LOAD_FORMAT_D16_X_gfx80       : MUBUF_Real_AllAddr_gfx80 <0x08>;
2265   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x09>;
2266   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0a>;
2267   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0b>;
2268   defm BUFFER_STORE_FORMAT_D16_X_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x0c>;
2269   defm BUFFER_STORE_FORMAT_D16_XY_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0d>;
2270   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0e>;
2271   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80   : MUBUF_Real_AllAddr_gfx80 <0x0f>;
2272 } // End HasUnpackedD16VMem.
2273 let SubtargetPredicate = HasPackedD16VMem in {
2274   defm BUFFER_LOAD_FORMAT_D16_X       : MUBUF_Real_AllAddr_vi <0x08>;
2275   defm BUFFER_LOAD_FORMAT_D16_XY      : MUBUF_Real_AllAddr_vi <0x09>;
2276   defm BUFFER_LOAD_FORMAT_D16_XYZ     : MUBUF_Real_AllAddr_vi <0x0a>;
2277   defm BUFFER_LOAD_FORMAT_D16_XYZW    : MUBUF_Real_AllAddr_vi <0x0b>;
2278   defm BUFFER_STORE_FORMAT_D16_X      : MUBUF_Real_AllAddr_vi <0x0c>;
2279   defm BUFFER_STORE_FORMAT_D16_XY     : MUBUF_Real_AllAddr_vi <0x0d>;
2280   defm BUFFER_STORE_FORMAT_D16_XYZ    : MUBUF_Real_AllAddr_vi <0x0e>;
2281   defm BUFFER_STORE_FORMAT_D16_XYZW   : MUBUF_Real_AllAddr_vi <0x0f>;
2282 } // End HasPackedD16VMem.
2283 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x10>;
2284 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x11>;
2285 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_Lds_vi <0x12>;
2286 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_Lds_vi <0x13>;
2287 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_Lds_vi <0x14>;
2288 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_Lds_vi <0x15>;
2289 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_Lds_vi <0x16>;
2290 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_Lds_vi <0x17>;
2291 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_vi <0x18>;
2292 defm BUFFER_STORE_BYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x19>;
2293 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_vi <0x1a>;
2294 defm BUFFER_STORE_SHORT_D16_HI  : MUBUF_Real_AllAddr_vi <0x1b>;
2295 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_vi <0x1c>;
2296 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_vi <0x1d>;
2297 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_vi <0x1e>;
2298 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_vi <0x1f>;
2299
2300 defm BUFFER_LOAD_UBYTE_D16      : MUBUF_Real_AllAddr_vi <0x20>;
2301 defm BUFFER_LOAD_UBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x21>;
2302 defm BUFFER_LOAD_SBYTE_D16      : MUBUF_Real_AllAddr_vi <0x22>;
2303 defm BUFFER_LOAD_SBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x23>;
2304 defm BUFFER_LOAD_SHORT_D16      : MUBUF_Real_AllAddr_vi <0x24>;
2305 defm BUFFER_LOAD_SHORT_D16_HI   : MUBUF_Real_AllAddr_vi <0x25>;
2306
2307 defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_vi <0x26>;
2308 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>;
2309
2310 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_vi <0x40>;
2311 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_vi <0x41>;
2312 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_vi <0x42>;
2313 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_vi <0x43>;
2314 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_vi <0x44>;
2315 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_vi <0x45>;
2316 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_vi <0x46>;
2317 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_vi <0x47>;
2318 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_vi <0x48>;
2319 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_vi <0x49>;
2320 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_vi <0x4a>;
2321 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_vi <0x4b>;
2322 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_vi <0x4c>;
2323
2324 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_vi <0x60>;
2325 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_vi <0x61>;
2326 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_vi <0x62>;
2327 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_vi <0x63>;
2328 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_vi <0x64>;
2329 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_vi <0x65>;
2330 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_vi <0x66>;
2331 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_vi <0x67>;
2332 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_vi <0x68>;
2333 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_vi <0x69>;
2334 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_vi <0x6a>;
2335 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_vi <0x6b>;
2336 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_vi <0x6c>;
2337
2338 def BUFFER_STORE_LDS_DWORD_vi   : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>;
2339
2340 def BUFFER_WBINVL1_vi           : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>;
2341 def BUFFER_WBINVL1_VOL_vi       : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>;
2342
2343 let SubtargetPredicate = HasAtomicFaddInsts in {
2344
2345 defm BUFFER_ATOMIC_ADD_F32    : MUBUF_Real_AllAddr_vi <0x4d>;
2346 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_AllAddr_vi <0x4e>;
2347
2348 } // End SubtargetPredicate = HasAtomicFaddInsts
2349
2350 class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
2351   MTBUF_Real<ps>,
2352   Enc64,
2353   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
2354   let AssemblerPredicate = isGFX8GFX9;
2355   let DecoderNamespace = "GFX8";
2356
2357   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2358   let Inst{12}    = ps.offen;
2359   let Inst{13}    = ps.idxen;
2360   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2361   let Inst{18-15} = op;
2362   let Inst{22-19} = dfmt;
2363   let Inst{25-23} = nfmt;
2364   let Inst{31-26} = 0x3a; //encoding
2365   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2366   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2367   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2368   let Inst{54}    = !if(ps.has_slc, slc, ?);
2369   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2370   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2371 }
2372
2373 multiclass MTBUF_Real_AllAddr_vi<bits<4> op> {
2374   def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2375   def _OFFEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2376   def _IDXEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2377   def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2378 }
2379
2380 class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> :
2381   MTBUF_Real<ps>,
2382   Enc64,
2383   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2384   let AssemblerPredicate=HasUnpackedD16VMem;
2385   let DecoderNamespace="GFX80_UNPACKED";
2386
2387   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2388   let Inst{12}    = ps.offen;
2389   let Inst{13}    = ps.idxen;
2390   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2391   let Inst{18-15} = op;
2392   let Inst{22-19} = dfmt;
2393   let Inst{25-23} = nfmt;
2394   let Inst{31-26} = 0x3a; //encoding
2395   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2396   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2397   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2398   let Inst{54}    = !if(ps.has_slc, slc, ?);
2399   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2400   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2401 }
2402
2403 multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> {
2404   def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2405   def _OFFEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2406   def _IDXEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2407   def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2408 }
2409
2410 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_vi <0x00>;
2411 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_vi <0x01>;
2412 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_vi <0x02>;
2413 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_vi <0x03>;
2414 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_vi <0x04>;
2415 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_vi <0x05>;
2416 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_vi <0x06>;
2417 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>;
2418 let SubtargetPredicate = HasUnpackedD16VMem in {
2419   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Real_AllAddr_gfx80 <0x08>;
2420   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x09>;
2421   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0a>;
2422   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0b>;
2423   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x0c>;
2424   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0d>;
2425   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0e>;
2426   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>;
2427 } // End HasUnpackedD16VMem.
2428 let SubtargetPredicate = HasPackedD16VMem in {
2429   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_vi <0x08>;
2430   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_vi <0x09>;
2431   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_vi <0x0a>;
2432   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_vi <0x0b>;
2433   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_vi <0x0c>;
2434   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_vi <0x0d>;
2435   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_vi <0x0e>;
2436   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>;
2437 } // End HasUnpackedD16VMem.
2438
2439 def MUBUFInfoTable : GenericTable {
2440   let FilterClass = "MUBUF_Pseudo";
2441   let CppTypeName = "MUBUFInfo";
2442   let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"];
2443
2444   let PrimaryKey = ["Opcode"];
2445   let PrimaryKeyName = "getMUBUFOpcodeHelper";
2446 }
2447
2448 def getMUBUFInfoFromOpcode : SearchIndex {
2449   let Table = MUBUFInfoTable;
2450   let Key = ["Opcode"];
2451 }
2452
2453 def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex {
2454   let Table = MUBUFInfoTable;
2455   let Key = ["BaseOpcode", "elements"];
2456 }
2457
2458 def MTBUFInfoTable : GenericTable {
2459   let FilterClass = "MTBUF_Pseudo";
2460   let CppTypeName = "MTBUFInfo";
2461   let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"];
2462
2463   let PrimaryKey = ["Opcode"];
2464   let PrimaryKeyName = "getMTBUFOpcodeHelper";
2465 }
2466
2467 def getMTBUFInfoFromOpcode : SearchIndex {
2468   let Table = MTBUFInfoTable;
2469   let Key = ["Opcode"];
2470 }
2471
2472 def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex {
2473   let Table = MTBUFInfoTable;
2474   let Key = ["BaseOpcode", "elements"];
2475 }