1 //===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">;
11 def MUBUFAddr64 : ComplexPattern<i64, 7, "SelectMUBUFAddr64">;
12 def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">;
14 def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantRoot]>;
15 def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantRoot], 20>;
17 def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">;
18 def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
19 def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">;
20 def MUBUFIntrinsicOffset : ComplexPattern<i32, 2, "SelectMUBUFIntrinsicOffset">;
21 def MUBUFIntrinsicVOffset : ComplexPattern<i32, 3, "SelectMUBUFIntrinsicVOffset">;
23 class MubufLoad <SDPatternOperator op> : PatFrag <
24 (ops node:$ptr), (op node:$ptr), [{
25 auto const AS = cast<MemSDNode>(N)->getAddressSpace();
26 return AS == AMDGPUASI.GLOBAL_ADDRESS ||
27 AS == AMDGPUASI.CONSTANT_ADDRESS;
30 def mubuf_load : MubufLoad <load>;
31 def mubuf_az_extloadi8 : MubufLoad <az_extloadi8>;
32 def mubuf_sextloadi8 : MubufLoad <sextloadi8>;
33 def mubuf_az_extloadi16 : MubufLoad <az_extloadi16>;
34 def mubuf_sextloadi16 : MubufLoad <sextloadi16>;
35 def mubuf_load_atomic : MubufLoad <atomic_load>;
45 class getAddrName<int addrKind> {
47 !if(!eq(addrKind, BUFAddrKind.Offset), "offset",
48 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen",
49 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen",
50 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",
51 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",
55 class MUBUFAddr64Table <bit is_addr64, string suffix = ""> {
56 bit IsAddr64 = is_addr64;
57 string OpName = NAME # suffix;
60 class MTBUFAddr64Table <bit is_addr64, string suffix = ""> {
61 bit IsAddr64 = is_addr64;
62 string OpName = NAME # suffix;
65 //===----------------------------------------------------------------------===//
67 //===----------------------------------------------------------------------===//
69 class MTBUF_Pseudo <string opName, dag outs, dag ins,
70 string asmOps, list<dag> pattern=[]> :
71 InstSI<outs, ins, "", pattern>,
72 SIMCInstr<opName, SIEncodingFamily.NONE> {
75 let isCodeGenOnly = 1;
77 let UseNamedOperandTable = 1;
79 string Mnemonic = opName;
80 string AsmOperands = asmOps;
86 let hasSideEffects = 0;
87 let SchedRW = [WriteVMEM];
89 let AsmMatchConverter = "cvtMtbuf";
94 bits<1> has_vdata = 1;
95 bits<1> has_vaddr = 1;
97 bits<1> glc_value = 0; // the value for glc if no such operand
98 bits<4> dfmt_value = 1; // the value for dfmt if no such operand
99 bits<3> nfmt_value = 0; // the value for nfmt if no such operand
100 bits<1> has_srsrc = 1;
101 bits<1> has_soffset = 1;
102 bits<1> has_offset = 1;
105 bits<1> has_dfmt = 1;
106 bits<1> has_nfmt = 1;
109 class MTBUF_Real <MTBUF_Pseudo ps> :
110 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
113 let isCodeGenOnly = 0;
115 // copy relevant pseudo op flags
116 let SubtargetPredicate = ps.SubtargetPredicate;
117 let AsmMatchConverter = ps.AsmMatchConverter;
118 let Constraints = ps.Constraints;
119 let DisableEncoding = ps.DisableEncoding;
120 let TSFlags = ps.TSFlags;
134 class getMTBUFInsDA<list<RegisterClass> vdataList,
135 list<RegisterClass> vaddrList=[]> {
136 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
137 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
138 dag InsNoData = !if(!empty(vaddrList),
139 (ins SReg_128:$srsrc, SCSrc_b32:$soffset,
140 offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, slc:$slc, tfe:$tfe),
141 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
142 offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, slc:$slc, tfe:$tfe)
144 dag InsData = !if(!empty(vaddrList),
145 (ins vdataClass:$vdata, SReg_128:$srsrc,
146 SCSrc_b32:$soffset, offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc,
148 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
149 SCSrc_b32:$soffset, offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc,
152 dag ret = !if(!empty(vdataList), InsNoData, InsData);
155 class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
157 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret,
158 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
159 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
160 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
161 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
165 class getMTBUFAsmOps<int addrKind> {
167 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $dfmt, $nfmt, $soffset",
168 !if(!eq(addrKind, BUFAddrKind.OffEn),
169 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset offen",
170 !if(!eq(addrKind, BUFAddrKind.IdxEn),
171 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset idxen",
172 !if(!eq(addrKind, BUFAddrKind.BothEn),
173 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset idxen offen",
174 !if(!eq(addrKind, BUFAddrKind.Addr64),
175 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset addr64",
177 string ret = Pfx # "$offset";
180 class MTBUF_SetupAddr<int addrKind> {
181 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
182 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
184 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
185 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
187 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
189 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
192 class MTBUF_Load_Pseudo <string opName,
194 RegisterClass vdataClass,
195 list<dag> pattern=[],
196 // Workaround bug bz30254
197 int addrKindCopy = addrKind>
198 : MTBUF_Pseudo<opName,
199 (outs vdataClass:$vdata),
200 getMTBUFIns<addrKindCopy>.ret,
201 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
203 MTBUF_SetupAddr<addrKindCopy> {
204 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
209 multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
210 ValueType load_vt = i32,
211 SDPatternOperator ld = null_frag> {
213 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
214 [(set load_vt:$vdata,
215 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$dfmt,
216 i8:$nfmt, i1:$glc, i1:$slc, i1:$tfe)))]>,
219 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
220 [(set load_vt:$vdata,
221 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset,
222 i8:$dfmt, i8:$nfmt, i1:$glc, i1:$slc, i1:$tfe)))]>,
225 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
226 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
227 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
229 let DisableWQM = 1 in {
230 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
231 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
232 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
233 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
237 class MTBUF_Store_Pseudo <string opName,
239 RegisterClass vdataClass,
240 list<dag> pattern=[],
241 // Workaround bug bz30254
242 int addrKindCopy = addrKind,
243 RegisterClass vdataClassCopy = vdataClass>
244 : MTBUF_Pseudo<opName,
246 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
247 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
249 MTBUF_SetupAddr<addrKindCopy> {
250 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
255 multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
256 ValueType store_vt = i32,
257 SDPatternOperator st = null_frag> {
259 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
260 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
261 i16:$offset, i8:$dfmt, i8:$nfmt, i1:$glc,
262 i1:$slc, i1:$tfe))]>,
265 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
266 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
267 i16:$offset, i8:$dfmt, i8:$nfmt, i1:$glc,
268 i1:$slc, i1:$tfe))]>,
271 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
272 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
273 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
275 let DisableWQM = 1 in {
276 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
277 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
278 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
279 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
284 //===----------------------------------------------------------------------===//
286 //===----------------------------------------------------------------------===//
288 class MUBUF_Pseudo <string opName, dag outs, dag ins,
289 string asmOps, list<dag> pattern=[]> :
290 InstSI<outs, ins, "", pattern>,
291 SIMCInstr<opName, SIEncodingFamily.NONE> {
294 let isCodeGenOnly = 1;
296 let UseNamedOperandTable = 1;
298 string Mnemonic = opName;
299 string AsmOperands = asmOps;
305 let hasSideEffects = 0;
306 let SchedRW = [WriteVMEM];
308 let AsmMatchConverter = "cvtMubuf";
313 bits<1> has_vdata = 1;
314 bits<1> has_vaddr = 1;
316 bits<1> glc_value = 0; // the value for glc if no such operand
317 bits<1> has_srsrc = 1;
318 bits<1> has_soffset = 1;
319 bits<1> has_offset = 1;
324 class MUBUF_Real <bits<7> op, MUBUF_Pseudo ps> :
325 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
328 let isCodeGenOnly = 0;
330 // copy relevant pseudo op flags
331 let SubtargetPredicate = ps.SubtargetPredicate;
332 let AsmMatchConverter = ps.AsmMatchConverter;
333 let Constraints = ps.Constraints;
334 let DisableEncoding = ps.DisableEncoding;
335 let TSFlags = ps.TSFlags;
349 // For cache invalidation instructions.
350 class MUBUF_Invalidate <string opName, SDPatternOperator node> :
351 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {
353 let AsmMatchConverter = "";
355 let hasSideEffects = 1;
358 // Set everything to 0.
373 class getMUBUFInsDA<list<RegisterClass> vdataList,
374 list<RegisterClass> vaddrList=[]> {
375 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
376 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
377 dag InsNoData = !if(!empty(vaddrList),
378 (ins SReg_128:$srsrc, SCSrc_b32:$soffset,
379 offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe),
380 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
381 offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe)
383 dag InsData = !if(!empty(vaddrList),
384 (ins vdataClass:$vdata, SReg_128:$srsrc,
385 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe),
386 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
387 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, slc:$slc, tfe:$tfe)
389 dag ret = !if(!empty(vdataList), InsNoData, InsData);
392 class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
394 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList>.ret,
395 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32]>.ret,
396 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32]>.ret,
397 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64]>.ret,
398 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64]>.ret,
402 class getMUBUFAsmOps<int addrKind> {
404 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",
405 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen",
406 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen",
407 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",
408 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",
410 string ret = Pfx # "$offset";
413 class MUBUF_SetupAddr<int addrKind> {
414 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
415 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
417 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
418 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
420 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
422 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
425 class MUBUF_Load_Pseudo <string opName,
427 RegisterClass vdataClass,
428 list<dag> pattern=[],
429 // Workaround bug bz30254
430 int addrKindCopy = addrKind>
431 : MUBUF_Pseudo<opName,
432 (outs vdataClass:$vdata),
433 getMUBUFIns<addrKindCopy>.ret,
434 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
436 MUBUF_SetupAddr<addrKindCopy> {
437 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
442 // FIXME: tfe can't be an operand because it requires a separate
443 // opcode because it needs an N+1 register class dest register.
444 multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
445 ValueType load_vt = i32,
446 SDPatternOperator ld = null_frag> {
448 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
449 [(set load_vt:$vdata,
450 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>,
453 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
454 [(set load_vt:$vdata,
455 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))]>,
458 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
459 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
460 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
462 let DisableWQM = 1 in {
463 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
464 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
465 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
466 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
470 class MUBUF_Store_Pseudo <string opName,
472 RegisterClass vdataClass,
473 list<dag> pattern=[],
474 // Workaround bug bz30254
475 int addrKindCopy = addrKind,
476 RegisterClass vdataClassCopy = vdataClass>
477 : MUBUF_Pseudo<opName,
479 getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
480 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe",
482 MUBUF_SetupAddr<addrKindCopy> {
483 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
488 multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
489 ValueType store_vt = i32,
490 SDPatternOperator st = null_frag> {
492 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
493 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
494 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>,
497 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
498 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
499 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>,
502 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
503 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
504 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
506 let DisableWQM = 1 in {
507 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
508 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
509 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
510 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
515 class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
516 list<RegisterClass> vaddrList=[]> {
517 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
518 dag ret = !if(vdata_in,
519 !if(!empty(vaddrList),
520 (ins vdataClass:$vdata_in,
521 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc),
522 (ins vdataClass:$vdata_in, vaddrClass:$vaddr,
523 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc)
525 !if(!empty(vaddrList),
526 (ins vdataClass:$vdata,
527 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc),
528 (ins vdataClass:$vdata, vaddrClass:$vaddr,
529 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, slc:$slc)
533 class getMUBUFAtomicIns<int addrKind,
534 RegisterClass vdataClass,
536 // Workaround bug bz30254
537 RegisterClass vdataClassCopy=vdataClass> {
539 !if(!eq(addrKind, BUFAddrKind.Offset),
540 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret,
541 !if(!eq(addrKind, BUFAddrKind.OffEn),
542 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
543 !if(!eq(addrKind, BUFAddrKind.IdxEn),
544 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
545 !if(!eq(addrKind, BUFAddrKind.BothEn),
546 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
547 !if(!eq(addrKind, BUFAddrKind.Addr64),
548 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
552 class MUBUF_Atomic_Pseudo<string opName,
557 list<dag> pattern=[],
558 // Workaround bug bz30254
559 int addrKindCopy = addrKind>
560 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,
561 MUBUF_SetupAddr<addrKindCopy> {
564 let hasPostISelHook = 1;
565 let hasSideEffects = 1;
571 class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
572 RegisterClass vdataClass,
573 list<dag> pattern=[],
574 // Workaround bug bz30254
575 int addrKindCopy = addrKind,
576 RegisterClass vdataClassCopy = vdataClass>
577 : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
579 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret,
580 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc",
582 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> {
583 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
585 let AsmMatchConverter = "cvtMubufAtomic";
588 class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
589 RegisterClass vdataClass,
590 list<dag> pattern=[],
591 // Workaround bug bz30254
592 int addrKindCopy = addrKind,
593 RegisterClass vdataClassCopy = vdataClass>
594 : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
595 (outs vdataClassCopy:$vdata),
596 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret,
597 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc",
599 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> {
600 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret;
602 let Constraints = "$vdata = $vdata_in";
603 let DisableEncoding = "$vdata_in";
604 let AsmMatchConverter = "cvtMubufAtomicReturn";
607 multiclass MUBUF_Pseudo_Atomics <string opName,
608 RegisterClass vdataClass,
610 SDPatternOperator atomic> {
612 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>,
613 MUBUFAddr64Table <0>;
614 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>,
615 MUBUFAddr64Table <1>;
616 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
617 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
618 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
620 def _RTN_OFFSET : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
621 [(set vdataType:$vdata,
622 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc),
623 vdataType:$vdata_in))]>,
624 MUBUFAddr64Table <0, "_RTN">;
626 def _RTN_ADDR64 : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
627 [(set vdataType:$vdata,
628 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc),
629 vdataType:$vdata_in))]>,
630 MUBUFAddr64Table <1, "_RTN">;
632 def _RTN_OFFEN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
633 def _RTN_IDXEN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
634 def _RTN_BOTHEN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
638 //===----------------------------------------------------------------------===//
639 // MUBUF Instructions
640 //===----------------------------------------------------------------------===//
642 let SubtargetPredicate = isGCN in {
644 defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads <
645 "buffer_load_format_x", VGPR_32
647 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
648 "buffer_load_format_xy", VReg_64
650 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <
651 "buffer_load_format_xyz", VReg_96
653 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <
654 "buffer_load_format_xyzw", VReg_128
656 defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <
657 "buffer_store_format_x", VGPR_32
659 defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <
660 "buffer_store_format_xy", VReg_64
662 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <
663 "buffer_store_format_xyz", VReg_96
665 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <
666 "buffer_store_format_xyzw", VReg_128
668 defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads <
669 "buffer_load_ubyte", VGPR_32, i32, mubuf_az_extloadi8
671 defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads <
672 "buffer_load_sbyte", VGPR_32, i32, mubuf_sextloadi8
674 defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads <
675 "buffer_load_ushort", VGPR_32, i32, mubuf_az_extloadi16
677 defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads <
678 "buffer_load_sshort", VGPR_32, i32, mubuf_sextloadi16
680 defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads <
681 "buffer_load_dword", VGPR_32, i32, mubuf_load
683 defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
684 "buffer_load_dwordx2", VReg_64, v2i32, mubuf_load
686 defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
687 "buffer_load_dwordx3", VReg_96, untyped, mubuf_load
689 defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
690 "buffer_load_dwordx4", VReg_128, v4i32, mubuf_load
692 defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
693 "buffer_store_byte", VGPR_32, i32, truncstorei8_global
695 defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
696 "buffer_store_short", VGPR_32, i32, truncstorei16_global
698 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
699 "buffer_store_dword", VGPR_32, i32, global_store
701 defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
702 "buffer_store_dwordx2", VReg_64, v2i32, global_store
704 defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
705 "buffer_store_dwordx3", VReg_96, untyped, global_store
707 defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
708 "buffer_store_dwordx4", VReg_128, v4i32, global_store
710 defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
711 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global
713 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <
714 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag
716 defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <
717 "buffer_atomic_add", VGPR_32, i32, atomic_add_global
719 defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <
720 "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global
722 defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <
723 "buffer_atomic_smin", VGPR_32, i32, atomic_min_global
725 defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <
726 "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global
728 defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <
729 "buffer_atomic_smax", VGPR_32, i32, atomic_max_global
731 defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <
732 "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global
734 defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <
735 "buffer_atomic_and", VGPR_32, i32, atomic_and_global
737 defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <
738 "buffer_atomic_or", VGPR_32, i32, atomic_or_global
740 defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <
741 "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global
743 defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <
744 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global
746 defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <
747 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global
749 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <
750 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global
752 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <
753 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag
755 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <
756 "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global
758 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <
759 "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global
761 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <
762 "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global
764 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <
765 "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global
767 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <
768 "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global
770 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <
771 "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global
773 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <
774 "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global
776 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <
777 "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global
779 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <
780 "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global
782 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <
783 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global
785 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
786 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global
789 let SubtargetPredicate = isSI in { // isn't on CI & VI
791 defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
792 defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">;
793 defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">;
794 defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">;
795 defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
796 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">;
797 defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">;
798 defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">;
801 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
802 int_amdgcn_buffer_wbinvl1_sc>;
805 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
806 int_amdgcn_buffer_wbinvl1>;
808 //===----------------------------------------------------------------------===//
809 // MTBUF Instructions
810 //===----------------------------------------------------------------------===//
812 defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32>;
813 defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64>;
814 defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_128>;
815 defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128>;
816 defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32>;
817 defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64>;
818 defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_128>;
819 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>;
821 } // End let SubtargetPredicate = isGCN
823 let SubtargetPredicate = isCIVI in {
825 //===----------------------------------------------------------------------===//
826 // Instruction definitions for CI and newer.
827 //===----------------------------------------------------------------------===//
828 // Remaining instructions:
829 // BUFFER_LOAD_DWORDX3
830 // BUFFER_STORE_DWORDX3
832 def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
833 int_amdgcn_buffer_wbinvl1_vol>;
835 } // End let SubtargetPredicate = isCIVI
837 //===----------------------------------------------------------------------===//
839 //===----------------------------------------------------------------------===//
841 let Predicates = [isGCN] in {
843 // Offset in an 32-bit VGPR
845 (SIload_constant v4i32:$sbase, i32:$voff),
846 (BUFFER_LOAD_DWORD_OFFEN $voff, $sbase, (i32 0), 0, 0, 0, 0)
850 //===----------------------------------------------------------------------===//
851 // buffer_load/store_format patterns
852 //===----------------------------------------------------------------------===//
854 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
857 (vt (name v4i32:$rsrc, 0,
858 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
859 imm:$glc, imm:$slc)),
860 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
861 (as_i1imm $glc), (as_i1imm $slc), 0)
865 (vt (name v4i32:$rsrc, i32:$vindex,
866 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
867 imm:$glc, imm:$slc)),
868 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
869 (as_i1imm $glc), (as_i1imm $slc), 0)
873 (vt (name v4i32:$rsrc, 0,
874 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
875 imm:$glc, imm:$slc)),
876 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
877 (as_i1imm $glc), (as_i1imm $slc), 0)
881 (vt (name v4i32:$rsrc, i32:$vindex,
882 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
883 imm:$glc, imm:$slc)),
884 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
885 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
886 $rsrc, $soffset, (as_i16imm $offset),
887 (as_i1imm $glc), (as_i1imm $slc), 0)
891 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
892 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
893 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;
894 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">;
895 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">;
896 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">;
898 multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
901 (name vt:$vdata, v4i32:$rsrc, 0,
902 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
904 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset),
905 (as_i1imm $glc), (as_i1imm $slc), 0)
909 (name vt:$vdata, v4i32:$rsrc, i32:$vindex,
910 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
912 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
913 (as_i16imm $offset), (as_i1imm $glc),
918 (name vt:$vdata, v4i32:$rsrc, 0,
919 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
921 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
922 (as_i16imm $offset), (as_i1imm $glc),
927 (name vt:$vdata, v4i32:$rsrc, i32:$vindex,
928 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
930 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
932 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
933 $rsrc, $soffset, (as_i16imm $offset),
934 (as_i1imm $glc), (as_i1imm $slc), 0)
938 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
939 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
940 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
941 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, f32, "BUFFER_STORE_DWORD">;
942 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v2f32, "BUFFER_STORE_DWORDX2">;
943 defm : MUBUF_StoreIntrinsicPat<int_amdgcn_buffer_store, v4f32, "BUFFER_STORE_DWORDX4">;
945 //===----------------------------------------------------------------------===//
946 // buffer_atomic patterns
947 //===----------------------------------------------------------------------===//
949 multiclass BufferAtomicPatterns<SDPatternOperator name, string opcode> {
951 (name i32:$vdata_in, v4i32:$rsrc, 0,
952 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
954 (!cast<MUBUF_Pseudo>(opcode # _RTN_OFFSET) $vdata_in, $rsrc, $soffset,
955 (as_i16imm $offset), (as_i1imm $slc))
959 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex,
960 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
962 (!cast<MUBUF_Pseudo>(opcode # _RTN_IDXEN) $vdata_in, $vindex, $rsrc, $soffset,
963 (as_i16imm $offset), (as_i1imm $slc))
967 (name i32:$vdata_in, v4i32:$rsrc, 0,
968 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
970 (!cast<MUBUF_Pseudo>(opcode # _RTN_OFFEN) $vdata_in, $voffset, $rsrc, $soffset,
971 (as_i16imm $offset), (as_i1imm $slc))
975 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex,
976 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
978 (!cast<MUBUF_Pseudo>(opcode # _RTN_BOTHEN)
980 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
981 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc))
985 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_swap, "BUFFER_ATOMIC_SWAP">;
986 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_add, "BUFFER_ATOMIC_ADD">;
987 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_sub, "BUFFER_ATOMIC_SUB">;
988 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smin, "BUFFER_ATOMIC_SMIN">;
989 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umin, "BUFFER_ATOMIC_UMIN">;
990 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_smax, "BUFFER_ATOMIC_SMAX">;
991 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_umax, "BUFFER_ATOMIC_UMAX">;
992 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_and, "BUFFER_ATOMIC_AND">;
993 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_or, "BUFFER_ATOMIC_OR">;
994 defm : BufferAtomicPatterns<int_amdgcn_buffer_atomic_xor, "BUFFER_ATOMIC_XOR">;
997 (int_amdgcn_buffer_atomic_cmpswap
998 i32:$data, i32:$cmp, v4i32:$rsrc, 0,
999 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
1002 (BUFFER_ATOMIC_CMPSWAP_RTN_OFFSET
1003 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1004 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
1009 (int_amdgcn_buffer_atomic_cmpswap
1010 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1011 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset),
1014 (BUFFER_ATOMIC_CMPSWAP_RTN_IDXEN
1015 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1016 $vindex, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
1021 (int_amdgcn_buffer_atomic_cmpswap
1022 i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1023 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
1026 (BUFFER_ATOMIC_CMPSWAP_RTN_OFFEN
1027 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1028 $voffset, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
1033 (int_amdgcn_buffer_atomic_cmpswap
1034 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1035 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset),
1038 (BUFFER_ATOMIC_CMPSWAP_RTN_BOTHEN
1039 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1040 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1041 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)),
1046 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
1047 PatFrag constant_ld> : Pat <
1048 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1049 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))),
1050 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe)
1053 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1054 ValueType vt, PatFrag atomic_ld> {
1056 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1057 i16:$offset, i1:$slc))),
1058 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 1, $slc, 0)
1062 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))),
1063 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 1, 0, 0)
1067 let Predicates = [isSICI] in {
1068 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
1069 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, az_extloadi8_constant>;
1070 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
1071 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, az_extloadi16_constant>;
1073 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, mubuf_load_atomic>;
1074 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, mubuf_load_atomic>;
1075 } // End Predicates = [isSICI]
1077 multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1081 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1082 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))),
1083 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe)
1087 let Predicates = [Has16BitInsts] in {
1089 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>;
1090 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_constant>;
1091 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, mubuf_sextloadi8>;
1092 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, mubuf_az_extloadi8>;
1094 } // End Predicates = [Has16BitInsts]
1096 multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen,
1097 MUBUF_Pseudo InstrOffset,
1098 ValueType vt, PatFrag ld> {
1100 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1101 i32:$soffset, u16imm:$offset))),
1102 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
1106 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))),
1107 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0)
1111 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>;
1112 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>;
1113 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>;
1114 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>;
1115 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>;
1116 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>;
1117 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>;
1118 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>;
1119 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>;
1121 // BUFFER_LOAD_DWORD*, addr64=0
1122 multiclass MUBUF_Load_Dword <ValueType vt,
1123 MUBUF_Pseudo offset,
1126 MUBUF_Pseudo bothen> {
1129 (vt (int_SI_buffer_load_dword v4i32:$rsrc, (i32 imm), i32:$soffset,
1130 imm:$offset, 0, 0, imm:$glc, imm:$slc,
1132 (offset $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc),
1133 (as_i1imm $slc), (as_i1imm $tfe))
1137 (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset,
1138 imm:$offset, 1, 0, imm:$glc, imm:$slc,
1140 (offen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc),
1145 (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset,
1146 imm:$offset, 0, 1, imm:$glc, imm:$slc,
1148 (idxen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc),
1149 (as_i1imm $slc), (as_i1imm $tfe))
1153 (vt (int_SI_buffer_load_dword v4i32:$rsrc, v2i32:$vaddr, i32:$soffset,
1154 imm:$offset, 1, 1, imm:$glc, imm:$slc,
1156 (bothen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc),
1161 defm : MUBUF_Load_Dword <i32, BUFFER_LOAD_DWORD_OFFSET, BUFFER_LOAD_DWORD_OFFEN,
1162 BUFFER_LOAD_DWORD_IDXEN, BUFFER_LOAD_DWORD_BOTHEN>;
1163 defm : MUBUF_Load_Dword <v2i32, BUFFER_LOAD_DWORDX2_OFFSET, BUFFER_LOAD_DWORDX2_OFFEN,
1164 BUFFER_LOAD_DWORDX2_IDXEN, BUFFER_LOAD_DWORDX2_BOTHEN>;
1165 defm : MUBUF_Load_Dword <v4i32, BUFFER_LOAD_DWORDX4_OFFSET, BUFFER_LOAD_DWORDX4_OFFEN,
1166 BUFFER_LOAD_DWORDX4_IDXEN, BUFFER_LOAD_DWORDX4_BOTHEN>;
1168 multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1169 ValueType vt, PatFrag atomic_st> {
1170 // Store follows atomic op convention so address is forst
1172 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1173 i16:$offset, i1:$slc), vt:$val),
1174 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 1, $slc, 0)
1178 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
1179 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 1, 0, 0)
1182 let Predicates = [isSICI] in {
1183 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, global_store_atomic>;
1184 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, global_store_atomic>;
1185 } // End Predicates = [isSICI]
1188 multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1192 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1193 i16:$offset, i1:$glc, i1:$slc, i1:$tfe)),
1194 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe)
1198 defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>;
1199 defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, global_store>;
1201 multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen,
1202 MUBUF_Pseudo InstrOffset,
1203 ValueType vt, PatFrag st> {
1205 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1206 i32:$soffset, u16imm:$offset)),
1207 (InstrOffen $value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
1211 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,
1213 (InstrOffset $value, $srsrc, $soffset, $offset, 0, 0, 0)
1217 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>;
1218 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>;
1219 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>;
1220 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>;
1221 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>;
1222 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private>;
1223 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private>;
1225 //===----------------------------------------------------------------------===//
1227 //===----------------------------------------------------------------------===//
1229 //===----------------------------------------------------------------------===//
1230 // tbuffer_load/store_format patterns
1231 //===----------------------------------------------------------------------===//
1233 multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1236 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1237 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)),
1238 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
1239 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1243 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1244 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)),
1245 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
1246 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1250 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1251 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)),
1252 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
1253 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1257 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset,
1258 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)),
1259 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN)
1260 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1261 $rsrc, $soffset, (as_i16imm $offset),
1262 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1266 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">;
1267 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;
1268 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">;
1269 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">;
1270 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;
1271 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;
1273 multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1276 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1277 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc),
1278 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset,
1279 (as_i16imm $offset), (as_i8imm $dfmt),
1280 (as_i8imm $nfmt), (as_i1imm $glc),
1285 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1286 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc),
1287 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
1288 (as_i16imm $offset), (as_i8imm $dfmt),
1289 (as_i8imm $nfmt), (as_i1imm $glc),
1294 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1295 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc),
1296 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
1297 (as_i16imm $offset), (as_i8imm $dfmt),
1298 (as_i8imm $nfmt), (as_i1imm $glc),
1303 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset,
1304 imm:$offset, imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc),
1305 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact)
1307 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1308 $rsrc, $soffset, (as_i16imm $offset),
1309 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0)
1313 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">;
1314 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;
1315 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4i32, "TBUFFER_STORE_FORMAT_XYZ">;
1316 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">;
1317 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">;
1318 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;
1319 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4f32, "TBUFFER_STORE_FORMAT_XYZ">;
1320 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;
1322 } // End let Predicates = [isGCN]
1324 //===----------------------------------------------------------------------===//
1325 // Target instructions, move to the appropriate target TD file
1326 //===----------------------------------------------------------------------===//
1328 //===----------------------------------------------------------------------===//
1330 //===----------------------------------------------------------------------===//
1332 class MUBUF_Real_si <bits<7> op, MUBUF_Pseudo ps> :
1335 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> {
1336 let AssemblerPredicate=isSICI;
1337 let DecoderNamespace="SICI";
1339 let Inst{11-0} = !if(ps.has_offset, offset, ?);
1340 let Inst{12} = ps.offen;
1341 let Inst{13} = ps.idxen;
1342 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value);
1343 let Inst{15} = ps.addr64;
1345 let Inst{24-18} = op;
1346 let Inst{31-26} = 0x38; //encoding
1347 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1348 let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1349 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1350 let Inst{54} = !if(ps.has_slc, slc, ?);
1351 let Inst{55} = !if(ps.has_tfe, tfe, ?);
1352 let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1355 multiclass MUBUF_Real_AllAddr_si<bits<7> op> {
1356 def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1357 def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>;
1358 def _OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1359 def _IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1360 def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1363 multiclass MUBUF_Real_Atomic_si<bits<7> op> : MUBUF_Real_AllAddr_si<op> {
1364 def _RTN_OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFSET")>;
1365 def _RTN_ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_ADDR64")>;
1366 def _RTN_OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFEN")>;
1367 def _RTN_IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_IDXEN")>;
1368 def _RTN_BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_BOTHEN")>;
1371 defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_si <0x00>;
1372 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_si <0x01>;
1373 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x02>;
1374 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x03>;
1375 defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_si <0x04>;
1376 defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_si <0x05>;
1377 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x06>;
1378 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x07>;
1379 defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_si <0x08>;
1380 defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_si <0x09>;
1381 defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_si <0x0a>;
1382 defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_si <0x0b>;
1383 defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_si <0x0c>;
1384 defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_si <0x0d>;
1385 defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_si <0x0e>;
1386 defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_si <0x0f>;
1387 defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_si <0x18>;
1388 defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_si <0x1a>;
1389 defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_si <0x1c>;
1390 defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_si <0x1d>;
1391 defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_si <0x1e>;
1392 defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_si <0x1f>;
1394 defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_si <0x30>;
1395 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_si <0x31>;
1396 defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_si <0x32>;
1397 defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_si <0x33>;
1398 //defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomic_si <0x34>; // isn't on CI & VI
1399 defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_si <0x35>;
1400 defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_si <0x36>;
1401 defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_si <0x37>;
1402 defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_si <0x38>;
1403 defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_si <0x39>;
1404 defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_si <0x3a>;
1405 defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_si <0x3b>;
1406 defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_si <0x3c>;
1407 defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_si <0x3d>;
1409 //defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_si <0x3e>; // isn't on VI
1410 //defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_si <0x3f>; // isn't on VI
1411 //defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_si <0x40>; // isn't on VI
1412 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_si <0x50>;
1413 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_si <0x51>;
1414 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_si <0x52>;
1415 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_si <0x53>;
1416 //defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomic_si <0x54>; // isn't on CI & VI
1417 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_si <0x55>;
1418 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_si <0x56>;
1419 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_si <0x57>;
1420 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_si <0x58>;
1421 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_si <0x59>;
1422 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_si <0x5a>;
1423 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_si <0x5b>;
1424 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_si <0x5c>;
1425 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_si <0x5d>;
1426 // FIXME: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on CI.
1427 //defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomic_si <0x5e">; // isn't on VI
1428 //defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomic_si <0x5f>; // isn't on VI
1429 //defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomic_si <0x60>; // isn't on VI
1431 def BUFFER_WBINVL1_SC_si : MUBUF_Real_si <0x70, BUFFER_WBINVL1_SC>;
1432 def BUFFER_WBINVL1_si : MUBUF_Real_si <0x71, BUFFER_WBINVL1>;
1434 class MTBUF_Real_si <bits<3> op, MTBUF_Pseudo ps> :
1437 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> {
1438 let AssemblerPredicate=isSICI;
1439 let DecoderNamespace="SICI";
1441 let Inst{11-0} = !if(ps.has_offset, offset, ?);
1442 let Inst{12} = ps.offen;
1443 let Inst{13} = ps.idxen;
1444 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value);
1445 let Inst{15} = ps.addr64;
1446 let Inst{18-16} = op;
1447 let Inst{22-19} = !if(ps.has_dfmt, dfmt, ps.dfmt_value);
1448 let Inst{25-23} = !if(ps.has_nfmt, nfmt, ps.nfmt_value);
1449 let Inst{31-26} = 0x3a; //encoding
1450 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1451 let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1452 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1453 let Inst{54} = !if(ps.has_slc, slc, ?);
1454 let Inst{55} = !if(ps.has_tfe, tfe, ?);
1455 let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1458 multiclass MTBUF_Real_AllAddr_si<bits<3> op> {
1459 def _OFFSET_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
1460 def _ADDR64_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>;
1461 def _OFFEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
1462 def _IDXEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
1463 def _BOTHEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
1466 defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_si <0>;
1467 defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_si <1>;
1468 //defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_si <2>;
1469 defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_si <3>;
1470 defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_si <4>;
1471 defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_si <5>;
1472 defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_si <6>;
1473 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_si <7>;
1475 //===----------------------------------------------------------------------===//
1477 //===----------------------------------------------------------------------===//
1479 class MUBUF_Real_ci <bits<7> op, MUBUF_Pseudo ps> :
1480 MUBUF_Real_si<op, ps> {
1481 let AssemblerPredicate=isCIOnly;
1482 let DecoderNamespace="CI";
1485 def BUFFER_WBINVL1_VOL_ci : MUBUF_Real_ci <0x70, BUFFER_WBINVL1_VOL>;
1488 //===----------------------------------------------------------------------===//
1490 //===----------------------------------------------------------------------===//
1492 class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> :
1495 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
1496 let AssemblerPredicate=isVI;
1497 let DecoderNamespace="VI";
1499 let Inst{11-0} = !if(ps.has_offset, offset, ?);
1500 let Inst{12} = ps.offen;
1501 let Inst{13} = ps.idxen;
1502 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value);
1504 let Inst{17} = !if(ps.has_slc, slc, ?);
1505 let Inst{24-18} = op;
1506 let Inst{31-26} = 0x38; //encoding
1507 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1508 let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1509 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1510 let Inst{55} = !if(ps.has_tfe, tfe, ?);
1511 let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1514 multiclass MUBUF_Real_AllAddr_vi<bits<7> op> {
1515 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1516 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1517 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1518 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1521 multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
1522 MUBUF_Real_AllAddr_vi<op> {
1523 def _RTN_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFSET")>;
1524 def _RTN_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_OFFEN")>;
1525 def _RTN_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_IDXEN")>;
1526 def _RTN_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_RTN_BOTHEN")>;
1529 defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_vi <0x00>;
1530 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>;
1531 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>;
1532 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>;
1533 defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>;
1534 defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>;
1535 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>;
1536 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>;
1537 defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_vi <0x10>;
1538 defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_vi <0x11>;
1539 defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_vi <0x12>;
1540 defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_vi <0x13>;
1541 defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_vi <0x14>;
1542 defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>;
1543 defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_vi <0x16>;
1544 defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_vi <0x17>;
1545 defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>;
1546 defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>;
1547 defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>;
1548 defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>;
1549 defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>;
1550 defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>;
1552 defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>;
1553 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>;
1554 defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>;
1555 defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>;
1556 defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>;
1557 defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>;
1558 defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>;
1559 defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>;
1560 defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>;
1561 defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>;
1562 defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>;
1563 defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>;
1564 defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>;
1566 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>;
1567 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>;
1568 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>;
1569 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>;
1570 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>;
1571 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>;
1572 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>;
1573 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>;
1574 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>;
1575 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>;
1576 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>;
1577 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>;
1578 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>;
1580 def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>;
1581 def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>;
1583 class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
1586 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
1587 let AssemblerPredicate=isVI;
1588 let DecoderNamespace="VI";
1590 let Inst{11-0} = !if(ps.has_offset, offset, ?);
1591 let Inst{12} = ps.offen;
1592 let Inst{13} = ps.idxen;
1593 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value);
1594 let Inst{18-15} = op;
1595 let Inst{22-19} = !if(ps.has_dfmt, dfmt, ps.dfmt_value);
1596 let Inst{25-23} = !if(ps.has_nfmt, nfmt, ps.nfmt_value);
1597 let Inst{31-26} = 0x3a; //encoding
1598 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1599 let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1600 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1601 let Inst{54} = !if(ps.has_slc, slc, ?);
1602 let Inst{55} = !if(ps.has_tfe, tfe, ?);
1603 let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1606 multiclass MTBUF_Real_AllAddr_vi<bits<4> op> {
1607 def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
1608 def _OFFEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
1609 def _IDXEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
1610 def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
1613 defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0>;
1614 defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <1>;
1615 //defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <2>;
1616 defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <3>;
1617 defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <4>;
1618 defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <5>;
1619 defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <6>;
1620 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <7>;