1 //===-- FLATInstructions.td - FLAT Instruction Defintions -----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 def FLATAtomic : ComplexPattern<i64, 3, "SelectFlatAtomic", [], [], -10>;
11 def FLATOffset : ComplexPattern<i64, 3, "SelectFlatOffset<false>", [], [], -10>;
13 def FLATOffsetSigned : ComplexPattern<i64, 3, "SelectFlatOffset<true>", [], [], -10>;
14 def FLATSignedAtomic : ComplexPattern<i64, 3, "SelectFlatAtomicSigned", [], [], -10>;
16 //===----------------------------------------------------------------------===//
18 //===----------------------------------------------------------------------===//
20 class FLAT_Pseudo<string opName, dag outs, dag ins,
21 string asmOps, list<dag> pattern=[]> :
22 InstSI<outs, ins, "", pattern>,
23 SIMCInstr<opName, SIEncodingFamily.NONE> {
26 let isCodeGenOnly = 1;
30 let UseNamedOperandTable = 1;
31 let hasSideEffects = 0;
32 let SchedRW = [WriteVMEM];
34 string Mnemonic = opName;
35 string AsmOperands = asmOps;
37 bits<1> is_flat_global = 0;
38 bits<1> is_flat_scratch = 0;
42 // We need to distinguish having saddr and enabling saddr because
43 // saddr is only valid for scratch and global instructions. Pre-gfx9
44 // these bits were reserved, so we also don't necessarily want to
45 // set these bits to the disabled value for the original flat
46 // segment instructions.
47 bits<1> has_saddr = 0;
48 bits<1> enabled_saddr = 0;
49 bits<7> saddr_value = 0;
50 bits<1> has_vaddr = 1;
56 let SubtargetPredicate = !if(is_flat_global, HasFlatGlobalInsts,
57 !if(is_flat_scratch, HasFlatScratchInsts, HasFlatAddressSpace));
59 // TODO: M0 if it could possibly access LDS (before gfx9? only)?
60 let Uses = !if(is_flat_global, [EXEC], [EXEC, FLAT_SCR]);
62 // Internally, FLAT instruction are executed as both an LDS and a
63 // Buffer instruction; so, they increment both VM_CNT and LGKM_CNT
64 // and are not considered done until both have been decremented.
66 let LGKM_CNT = !if(!or(is_flat_global, is_flat_scratch), 0, 1);
69 class FLAT_Real <bits<7> op, FLAT_Pseudo ps> :
70 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>,
74 let isCodeGenOnly = 0;
76 // copy relevant pseudo op flags
77 let SubtargetPredicate = ps.SubtargetPredicate;
78 let AsmMatchConverter = ps.AsmMatchConverter;
79 let TSFlags = ps.TSFlags;
80 let UseNamedOperandTable = ps.UseNamedOperandTable;
92 bits<1> lds = 0; // XXX - What does this actually do?
94 // Segment, 00=flat, 01=scratch, 10=global, 11=reserved
95 bits<2> seg = !if(ps.is_flat_global, 0b10,
96 !if(ps.is_flat_scratch, 0b01, 0));
98 // Signed offset. Highest bit ignored for flat and treated as 12-bit
99 // unsigned for flat acceses.
101 bits<1> nv = 0; // XXX - What does this actually do?
103 // We don't use tfe right now, and it was removed in gfx9.
106 // Only valid on GFX9+
107 let Inst{12-0} = offset;
109 let Inst{15-14} = seg;
111 let Inst{16} = !if(ps.has_glc, glc, ps.glcValue);
113 let Inst{24-18} = op;
114 let Inst{31-26} = 0x37; // Encoding.
115 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
116 let Inst{47-40} = !if(ps.has_data, vdata, ?);
117 let Inst{54-48} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7f), 0);
119 // 54-48 is reserved.
120 let Inst{55} = nv; // nv on GFX9+, TFE before.
121 let Inst{63-56} = !if(ps.has_vdst, vdst, ?);
124 class GlobalSaddrTable <bit is_saddr, string Name = ""> {
125 bit IsSaddr = is_saddr;
126 string SaddrOp = Name;
129 // TODO: Is exec allowed for saddr? The disabled value 0x7f is the
130 // same encoding value as exec_hi, so it isn't possible to use that if
131 // saddr is 32-bit (which isn't handled here yet).
132 class FLAT_Load_Pseudo <string opName, RegisterClass regClass,
133 bit HasTiedOutput = 0,
134 bit HasSignedOffset = 0, bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo<
136 (outs regClass:$vdst),
140 !con((ins VReg_64:$vaddr),
141 !if(EnableSaddr, (ins SReg_64:$saddr), (ins))),
142 (ins !if(HasSignedOffset,offset_s13,offset_u12):$offset)),
143 (ins GLC:$glc, SLC:$slc)),
144 !if(HasTiedOutput, (ins regClass:$vdst_in), (ins))),
145 " $vdst, $vaddr"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$glc$slc"> {
148 let has_saddr = HasSaddr;
149 let enabled_saddr = EnableSaddr;
150 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", "");
153 let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", "");
154 let DisableEncoding = !if(HasTiedOutput, "$vdst_in", "");
157 class FLAT_Store_Pseudo <string opName, RegisterClass vdataClass,
158 bit HasSignedOffset = 0, bit HasSaddr = 0, bit EnableSaddr = 0> : FLAT_Pseudo<
163 !con((ins VReg_64:$vaddr, vdataClass:$vdata),
164 !if(EnableSaddr, (ins SReg_64:$saddr), (ins))),
165 (ins !if(HasSignedOffset,offset_s13,offset_u12):$offset)),
166 (ins GLC:$glc, SLC:$slc)),
167 " $vaddr, $vdata"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$glc$slc"> {
171 let has_saddr = HasSaddr;
172 let enabled_saddr = EnableSaddr;
173 let PseudoInstr = opName#!if(!and(HasSaddr, EnableSaddr), "_SADDR", "");
177 multiclass FLAT_Global_Load_Pseudo<string opName, RegisterClass regClass, bit HasTiedInput = 0> {
178 let is_flat_global = 1 in {
179 def "" : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1>,
180 GlobalSaddrTable<0, opName>;
181 def _SADDR : FLAT_Load_Pseudo<opName, regClass, HasTiedInput, 1, 1, 1>,
182 GlobalSaddrTable<1, opName>;
186 multiclass FLAT_Global_Store_Pseudo<string opName, RegisterClass regClass> {
187 let is_flat_global = 1 in {
188 def "" : FLAT_Store_Pseudo<opName, regClass, 1, 1>,
189 GlobalSaddrTable<0, opName>;
190 def _SADDR : FLAT_Store_Pseudo<opName, regClass, 1, 1, 1>,
191 GlobalSaddrTable<1, opName>;
195 class FLAT_Scratch_Load_Pseudo <string opName, RegisterClass regClass,
196 bit EnableSaddr = 0>: FLAT_Pseudo<
198 (outs regClass:$vdst),
200 (ins SReg_32_XEXEC_HI:$saddr, offset_s13:$offset, GLC:$glc, SLC:$slc),
201 (ins VGPR_32:$vaddr, offset_s13:$offset, GLC:$glc, SLC:$slc)),
202 " $vdst, "#!if(EnableSaddr, "off", "$vaddr")#!if(EnableSaddr, ", $saddr", ", off")#"$offset$glc$slc"> {
206 let enabled_saddr = EnableSaddr;
207 let has_vaddr = !if(EnableSaddr, 0, 1);
208 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", "");
212 class FLAT_Scratch_Store_Pseudo <string opName, RegisterClass vdataClass, bit EnableSaddr = 0> : FLAT_Pseudo<
216 (ins vdataClass:$vdata, SReg_32_XEXEC_HI:$saddr, offset_s13:$offset, GLC:$glc, SLC:$slc),
217 (ins vdataClass:$vdata, VGPR_32:$vaddr, offset_s13:$offset, GLC:$glc, SLC:$slc)),
218 " "#!if(EnableSaddr, "off", "$vaddr")#", $vdata, "#!if(EnableSaddr, "$saddr", "off")#"$offset$glc$slc"> {
223 let enabled_saddr = EnableSaddr;
224 let has_vaddr = !if(EnableSaddr, 0, 1);
225 let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", "");
229 multiclass FLAT_Scratch_Load_Pseudo<string opName, RegisterClass regClass> {
230 let is_flat_scratch = 1 in {
231 def "" : FLAT_Scratch_Load_Pseudo<opName, regClass>;
232 def _SADDR : FLAT_Scratch_Load_Pseudo<opName, regClass, 1>;
236 multiclass FLAT_Scratch_Store_Pseudo<string opName, RegisterClass regClass> {
237 let is_flat_scratch = 1 in {
238 def "" : FLAT_Scratch_Store_Pseudo<opName, regClass>;
239 def _SADDR : FLAT_Scratch_Store_Pseudo<opName, regClass, 1>;
243 class FLAT_AtomicNoRet_Pseudo<string opName, dag outs, dag ins,
244 string asm, list<dag> pattern = []> :
245 FLAT_Pseudo<opName, outs, ins, asm, pattern> {
254 class FLAT_AtomicRet_Pseudo<string opName, dag outs, dag ins,
255 string asm, list<dag> pattern = []>
256 : FLAT_AtomicNoRet_Pseudo<opName, outs, ins, asm, pattern> {
257 let hasPostISelHook = 1;
260 let PseudoInstr = NAME # "_RTN";
263 multiclass FLAT_Atomic_Pseudo<
265 RegisterClass vdst_rc,
267 SDPatternOperator atomic = null_frag,
268 ValueType data_vt = vt,
269 RegisterClass data_rc = vdst_rc> {
270 def "" : FLAT_AtomicNoRet_Pseudo <opName,
272 (ins VReg_64:$vaddr, data_rc:$vdata, offset_u12:$offset, SLC:$slc),
273 " $vaddr, $vdata$offset$slc">,
274 GlobalSaddrTable<0, opName>,
275 AtomicNoRet <opName, 0> {
276 let PseudoInstr = NAME;
279 def _RTN : FLAT_AtomicRet_Pseudo <opName,
280 (outs vdst_rc:$vdst),
281 (ins VReg_64:$vaddr, data_rc:$vdata, offset_u12:$offset, SLC:$slc),
282 " $vdst, $vaddr, $vdata$offset glc$slc",
284 (atomic (FLATAtomic i64:$vaddr, i16:$offset, i1:$slc), data_vt:$vdata))]>,
285 GlobalSaddrTable<0, opName#"_rtn">,
286 AtomicNoRet <opName, 1>;
289 multiclass FLAT_Global_Atomic_Pseudo_NO_RTN<
291 RegisterClass vdst_rc,
293 SDPatternOperator atomic = null_frag,
294 ValueType data_vt = vt,
295 RegisterClass data_rc = vdst_rc> {
297 def "" : FLAT_AtomicNoRet_Pseudo <opName,
299 (ins VReg_64:$vaddr, data_rc:$vdata, offset_s13:$offset, SLC:$slc),
300 " $vaddr, $vdata, off$offset$slc">,
301 GlobalSaddrTable<0, opName>,
302 AtomicNoRet <opName, 0> {
304 let PseudoInstr = NAME;
307 def _SADDR : FLAT_AtomicNoRet_Pseudo <opName,
309 (ins VReg_64:$vaddr, data_rc:$vdata, SReg_64:$saddr, offset_s13:$offset, SLC:$slc),
310 " $vaddr, $vdata, $saddr$offset$slc">,
311 GlobalSaddrTable<1, opName>,
312 AtomicNoRet <opName#"_saddr", 0> {
314 let enabled_saddr = 1;
315 let PseudoInstr = NAME#"_SADDR";
319 multiclass FLAT_Global_Atomic_Pseudo_RTN<
321 RegisterClass vdst_rc,
323 SDPatternOperator atomic = null_frag,
324 ValueType data_vt = vt,
325 RegisterClass data_rc = vdst_rc> {
327 def _RTN : FLAT_AtomicRet_Pseudo <opName,
328 (outs vdst_rc:$vdst),
329 (ins VReg_64:$vaddr, data_rc:$vdata, offset_s13:$offset, SLC:$slc),
330 " $vdst, $vaddr, $vdata, off$offset glc$slc",
332 (atomic (FLATSignedAtomic i64:$vaddr, i16:$offset, i1:$slc), data_vt:$vdata))]>,
333 GlobalSaddrTable<0, opName#"_rtn">,
334 AtomicNoRet <opName, 1> {
338 def _SADDR_RTN : FLAT_AtomicRet_Pseudo <opName,
339 (outs vdst_rc:$vdst),
340 (ins VReg_64:$vaddr, data_rc:$vdata, SReg_64:$saddr, offset_s13:$offset, SLC:$slc),
341 " $vdst, $vaddr, $vdata, $saddr$offset glc$slc">,
342 GlobalSaddrTable<1, opName#"_rtn">,
343 AtomicNoRet <opName#"_saddr", 1> {
345 let enabled_saddr = 1;
346 let PseudoInstr = NAME#"_SADDR_RTN";
350 multiclass FLAT_Global_Atomic_Pseudo<
352 RegisterClass vdst_rc,
354 SDPatternOperator atomic = null_frag,
355 ValueType data_vt = vt,
356 RegisterClass data_rc = vdst_rc> :
357 FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, atomic, data_vt, data_rc>,
358 FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, atomic, data_vt, data_rc>;
360 class flat_binary_atomic_op<SDNode atomic_op> : PatFrag<
361 (ops node:$ptr, node:$value),
362 (atomic_op node:$ptr, node:$value),
363 [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::FLAT_ADDRESS;}]
366 def atomic_cmp_swap_flat : flat_binary_atomic_op<AMDGPUatomic_cmp_swap>;
367 def atomic_swap_flat : flat_binary_atomic_op<atomic_swap>;
368 def atomic_add_flat : flat_binary_atomic_op<atomic_load_add>;
369 def atomic_and_flat : flat_binary_atomic_op<atomic_load_and>;
370 def atomic_max_flat : flat_binary_atomic_op<atomic_load_max>;
371 def atomic_min_flat : flat_binary_atomic_op<atomic_load_min>;
372 def atomic_or_flat : flat_binary_atomic_op<atomic_load_or>;
373 def atomic_sub_flat : flat_binary_atomic_op<atomic_load_sub>;
374 def atomic_umax_flat : flat_binary_atomic_op<atomic_load_umax>;
375 def atomic_umin_flat : flat_binary_atomic_op<atomic_load_umin>;
376 def atomic_xor_flat : flat_binary_atomic_op<atomic_load_xor>;
377 def atomic_inc_flat : flat_binary_atomic_op<SIatomic_inc>;
378 def atomic_dec_flat : flat_binary_atomic_op<SIatomic_dec>;
382 //===----------------------------------------------------------------------===//
384 //===----------------------------------------------------------------------===//
386 def FLAT_LOAD_UBYTE : FLAT_Load_Pseudo <"flat_load_ubyte", VGPR_32>;
387 def FLAT_LOAD_SBYTE : FLAT_Load_Pseudo <"flat_load_sbyte", VGPR_32>;
388 def FLAT_LOAD_USHORT : FLAT_Load_Pseudo <"flat_load_ushort", VGPR_32>;
389 def FLAT_LOAD_SSHORT : FLAT_Load_Pseudo <"flat_load_sshort", VGPR_32>;
390 def FLAT_LOAD_DWORD : FLAT_Load_Pseudo <"flat_load_dword", VGPR_32>;
391 def FLAT_LOAD_DWORDX2 : FLAT_Load_Pseudo <"flat_load_dwordx2", VReg_64>;
392 def FLAT_LOAD_DWORDX4 : FLAT_Load_Pseudo <"flat_load_dwordx4", VReg_128>;
393 def FLAT_LOAD_DWORDX3 : FLAT_Load_Pseudo <"flat_load_dwordx3", VReg_96>;
395 def FLAT_STORE_BYTE : FLAT_Store_Pseudo <"flat_store_byte", VGPR_32>;
396 def FLAT_STORE_SHORT : FLAT_Store_Pseudo <"flat_store_short", VGPR_32>;
397 def FLAT_STORE_DWORD : FLAT_Store_Pseudo <"flat_store_dword", VGPR_32>;
398 def FLAT_STORE_DWORDX2 : FLAT_Store_Pseudo <"flat_store_dwordx2", VReg_64>;
399 def FLAT_STORE_DWORDX4 : FLAT_Store_Pseudo <"flat_store_dwordx4", VReg_128>;
400 def FLAT_STORE_DWORDX3 : FLAT_Store_Pseudo <"flat_store_dwordx3", VReg_96>;
402 let SubtargetPredicate = HasD16LoadStore in {
403 def FLAT_LOAD_UBYTE_D16 : FLAT_Load_Pseudo <"flat_load_ubyte_d16", VGPR_32, 1>;
404 def FLAT_LOAD_UBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_ubyte_d16_hi", VGPR_32, 1>;
405 def FLAT_LOAD_SBYTE_D16 : FLAT_Load_Pseudo <"flat_load_sbyte_d16", VGPR_32, 1>;
406 def FLAT_LOAD_SBYTE_D16_HI : FLAT_Load_Pseudo <"flat_load_sbyte_d16_hi", VGPR_32, 1>;
407 def FLAT_LOAD_SHORT_D16 : FLAT_Load_Pseudo <"flat_load_short_d16", VGPR_32, 1>;
408 def FLAT_LOAD_SHORT_D16_HI : FLAT_Load_Pseudo <"flat_load_short_d16_hi", VGPR_32, 1>;
410 def FLAT_STORE_BYTE_D16_HI : FLAT_Store_Pseudo <"flat_store_byte_d16_hi", VGPR_32>;
411 def FLAT_STORE_SHORT_D16_HI : FLAT_Store_Pseudo <"flat_store_short_d16_hi", VGPR_32>;
414 defm FLAT_ATOMIC_CMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap",
415 VGPR_32, i32, atomic_cmp_swap_flat,
418 defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap_x2",
419 VReg_64, i64, atomic_cmp_swap_flat,
422 defm FLAT_ATOMIC_SWAP : FLAT_Atomic_Pseudo <"flat_atomic_swap",
423 VGPR_32, i32, atomic_swap_flat>;
425 defm FLAT_ATOMIC_SWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_swap_x2",
426 VReg_64, i64, atomic_swap_flat>;
428 defm FLAT_ATOMIC_ADD : FLAT_Atomic_Pseudo <"flat_atomic_add",
429 VGPR_32, i32, atomic_add_flat>;
431 defm FLAT_ATOMIC_SUB : FLAT_Atomic_Pseudo <"flat_atomic_sub",
432 VGPR_32, i32, atomic_sub_flat>;
434 defm FLAT_ATOMIC_SMIN : FLAT_Atomic_Pseudo <"flat_atomic_smin",
435 VGPR_32, i32, atomic_min_flat>;
437 defm FLAT_ATOMIC_UMIN : FLAT_Atomic_Pseudo <"flat_atomic_umin",
438 VGPR_32, i32, atomic_umin_flat>;
440 defm FLAT_ATOMIC_SMAX : FLAT_Atomic_Pseudo <"flat_atomic_smax",
441 VGPR_32, i32, atomic_max_flat>;
443 defm FLAT_ATOMIC_UMAX : FLAT_Atomic_Pseudo <"flat_atomic_umax",
444 VGPR_32, i32, atomic_umax_flat>;
446 defm FLAT_ATOMIC_AND : FLAT_Atomic_Pseudo <"flat_atomic_and",
447 VGPR_32, i32, atomic_and_flat>;
449 defm FLAT_ATOMIC_OR : FLAT_Atomic_Pseudo <"flat_atomic_or",
450 VGPR_32, i32, atomic_or_flat>;
452 defm FLAT_ATOMIC_XOR : FLAT_Atomic_Pseudo <"flat_atomic_xor",
453 VGPR_32, i32, atomic_xor_flat>;
455 defm FLAT_ATOMIC_INC : FLAT_Atomic_Pseudo <"flat_atomic_inc",
456 VGPR_32, i32, atomic_inc_flat>;
458 defm FLAT_ATOMIC_DEC : FLAT_Atomic_Pseudo <"flat_atomic_dec",
459 VGPR_32, i32, atomic_dec_flat>;
461 defm FLAT_ATOMIC_ADD_X2 : FLAT_Atomic_Pseudo <"flat_atomic_add_x2",
462 VReg_64, i64, atomic_add_flat>;
464 defm FLAT_ATOMIC_SUB_X2 : FLAT_Atomic_Pseudo <"flat_atomic_sub_x2",
465 VReg_64, i64, atomic_sub_flat>;
467 defm FLAT_ATOMIC_SMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smin_x2",
468 VReg_64, i64, atomic_min_flat>;
470 defm FLAT_ATOMIC_UMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umin_x2",
471 VReg_64, i64, atomic_umin_flat>;
473 defm FLAT_ATOMIC_SMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smax_x2",
474 VReg_64, i64, atomic_max_flat>;
476 defm FLAT_ATOMIC_UMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umax_x2",
477 VReg_64, i64, atomic_umax_flat>;
479 defm FLAT_ATOMIC_AND_X2 : FLAT_Atomic_Pseudo <"flat_atomic_and_x2",
480 VReg_64, i64, atomic_and_flat>;
482 defm FLAT_ATOMIC_OR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_or_x2",
483 VReg_64, i64, atomic_or_flat>;
485 defm FLAT_ATOMIC_XOR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_xor_x2",
486 VReg_64, i64, atomic_xor_flat>;
488 defm FLAT_ATOMIC_INC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_inc_x2",
489 VReg_64, i64, atomic_inc_flat>;
491 defm FLAT_ATOMIC_DEC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_dec_x2",
492 VReg_64, i64, atomic_dec_flat>;
494 let SubtargetPredicate = isCI in { // CI Only flat instructions : FIXME Only?
496 defm FLAT_ATOMIC_FCMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap",
497 VGPR_32, f32, null_frag, v2f32, VReg_64>;
499 defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fcmpswap_x2",
500 VReg_64, f64, null_frag, v2f64, VReg_128>;
502 defm FLAT_ATOMIC_FMIN : FLAT_Atomic_Pseudo <"flat_atomic_fmin",
505 defm FLAT_ATOMIC_FMAX : FLAT_Atomic_Pseudo <"flat_atomic_fmax",
508 defm FLAT_ATOMIC_FMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmin_x2",
511 defm FLAT_ATOMIC_FMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_fmax_x2",
514 } // End SubtargetPredicate = isCI
516 let SubtargetPredicate = HasFlatGlobalInsts in {
517 defm GLOBAL_LOAD_UBYTE : FLAT_Global_Load_Pseudo <"global_load_ubyte", VGPR_32>;
518 defm GLOBAL_LOAD_SBYTE : FLAT_Global_Load_Pseudo <"global_load_sbyte", VGPR_32>;
519 defm GLOBAL_LOAD_USHORT : FLAT_Global_Load_Pseudo <"global_load_ushort", VGPR_32>;
520 defm GLOBAL_LOAD_SSHORT : FLAT_Global_Load_Pseudo <"global_load_sshort", VGPR_32>;
521 defm GLOBAL_LOAD_DWORD : FLAT_Global_Load_Pseudo <"global_load_dword", VGPR_32>;
522 defm GLOBAL_LOAD_DWORDX2 : FLAT_Global_Load_Pseudo <"global_load_dwordx2", VReg_64>;
523 defm GLOBAL_LOAD_DWORDX3 : FLAT_Global_Load_Pseudo <"global_load_dwordx3", VReg_96>;
524 defm GLOBAL_LOAD_DWORDX4 : FLAT_Global_Load_Pseudo <"global_load_dwordx4", VReg_128>;
526 defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16", VGPR_32, 1>;
527 defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_ubyte_d16_hi", VGPR_32, 1>;
528 defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16", VGPR_32, 1>;
529 defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Global_Load_Pseudo <"global_load_sbyte_d16_hi", VGPR_32, 1>;
530 defm GLOBAL_LOAD_SHORT_D16 : FLAT_Global_Load_Pseudo <"global_load_short_d16", VGPR_32, 1>;
531 defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Global_Load_Pseudo <"global_load_short_d16_hi", VGPR_32, 1>;
533 defm GLOBAL_STORE_BYTE : FLAT_Global_Store_Pseudo <"global_store_byte", VGPR_32>;
534 defm GLOBAL_STORE_SHORT : FLAT_Global_Store_Pseudo <"global_store_short", VGPR_32>;
535 defm GLOBAL_STORE_DWORD : FLAT_Global_Store_Pseudo <"global_store_dword", VGPR_32>;
536 defm GLOBAL_STORE_DWORDX2 : FLAT_Global_Store_Pseudo <"global_store_dwordx2", VReg_64>;
537 defm GLOBAL_STORE_DWORDX3 : FLAT_Global_Store_Pseudo <"global_store_dwordx3", VReg_96>;
538 defm GLOBAL_STORE_DWORDX4 : FLAT_Global_Store_Pseudo <"global_store_dwordx4", VReg_128>;
540 defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Global_Store_Pseudo <"global_store_byte_d16_hi", VGPR_32>;
541 defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d16_hi", VGPR_32>;
543 let is_flat_global = 1 in {
544 defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap",
545 VGPR_32, i32, AMDGPUatomic_cmp_swap_global,
548 defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2",
549 VReg_64, i64, AMDGPUatomic_cmp_swap_global,
552 defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap",
553 VGPR_32, i32, atomic_swap_global>;
555 defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_swap_x2",
556 VReg_64, i64, atomic_swap_global>;
558 defm GLOBAL_ATOMIC_ADD : FLAT_Global_Atomic_Pseudo <"global_atomic_add",
559 VGPR_32, i32, atomic_add_global>;
561 defm GLOBAL_ATOMIC_SUB : FLAT_Global_Atomic_Pseudo <"global_atomic_sub",
562 VGPR_32, i32, atomic_sub_global>;
564 defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_smin",
565 VGPR_32, i32, atomic_min_global>;
567 defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Atomic_Pseudo <"global_atomic_umin",
568 VGPR_32, i32, atomic_umin_global>;
570 defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_smax",
571 VGPR_32, i32, atomic_max_global>;
573 defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Atomic_Pseudo <"global_atomic_umax",
574 VGPR_32, i32, atomic_umax_global>;
576 defm GLOBAL_ATOMIC_AND : FLAT_Global_Atomic_Pseudo <"global_atomic_and",
577 VGPR_32, i32, atomic_and_global>;
579 defm GLOBAL_ATOMIC_OR : FLAT_Global_Atomic_Pseudo <"global_atomic_or",
580 VGPR_32, i32, atomic_or_global>;
582 defm GLOBAL_ATOMIC_XOR : FLAT_Global_Atomic_Pseudo <"global_atomic_xor",
583 VGPR_32, i32, atomic_xor_global>;
585 defm GLOBAL_ATOMIC_INC : FLAT_Global_Atomic_Pseudo <"global_atomic_inc",
586 VGPR_32, i32, atomic_inc_global>;
588 defm GLOBAL_ATOMIC_DEC : FLAT_Global_Atomic_Pseudo <"global_atomic_dec",
589 VGPR_32, i32, atomic_dec_global>;
591 defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_add_x2",
592 VReg_64, i64, atomic_add_global>;
594 defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_sub_x2",
595 VReg_64, i64, atomic_sub_global>;
597 defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smin_x2",
598 VReg_64, i64, atomic_min_global>;
600 defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umin_x2",
601 VReg_64, i64, atomic_umin_global>;
603 defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_smax_x2",
604 VReg_64, i64, atomic_max_global>;
606 defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_umax_x2",
607 VReg_64, i64, atomic_umax_global>;
609 defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_and_x2",
610 VReg_64, i64, atomic_and_global>;
612 defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_or_x2",
613 VReg_64, i64, atomic_or_global>;
615 defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_xor_x2",
616 VReg_64, i64, atomic_xor_global>;
618 defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_inc_x2",
619 VReg_64, i64, atomic_inc_global>;
621 defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_dec_x2",
622 VReg_64, i64, atomic_dec_global>;
623 } // End is_flat_global = 1
625 } // End SubtargetPredicate = HasFlatGlobalInsts
628 let SubtargetPredicate = HasFlatScratchInsts in {
629 defm SCRATCH_LOAD_UBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte", VGPR_32>;
630 defm SCRATCH_LOAD_SBYTE : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>;
631 defm SCRATCH_LOAD_USHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort", VGPR_32>;
632 defm SCRATCH_LOAD_SSHORT : FLAT_Scratch_Load_Pseudo <"scratch_load_sshort", VGPR_32>;
633 defm SCRATCH_LOAD_DWORD : FLAT_Scratch_Load_Pseudo <"scratch_load_dword", VGPR_32>;
634 defm SCRATCH_LOAD_DWORDX2 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx2", VReg_64>;
635 defm SCRATCH_LOAD_DWORDX3 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx3", VReg_96>;
636 defm SCRATCH_LOAD_DWORDX4 : FLAT_Scratch_Load_Pseudo <"scratch_load_dwordx4", VReg_128>;
638 defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16", VGPR_32>;
639 defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte_d16_hi", VGPR_32>;
640 defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16", VGPR_32>;
641 defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte_d16_hi", VGPR_32>;
642 defm SCRATCH_LOAD_SHORT_D16 : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16", VGPR_32>;
643 defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Scratch_Load_Pseudo <"scratch_load_short_d16_hi", VGPR_32>;
645 defm SCRATCH_STORE_BYTE : FLAT_Scratch_Store_Pseudo <"scratch_store_byte", VGPR_32>;
646 defm SCRATCH_STORE_SHORT : FLAT_Scratch_Store_Pseudo <"scratch_store_short", VGPR_32>;
647 defm SCRATCH_STORE_DWORD : FLAT_Scratch_Store_Pseudo <"scratch_store_dword", VGPR_32>;
648 defm SCRATCH_STORE_DWORDX2 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx2", VReg_64>;
649 defm SCRATCH_STORE_DWORDX3 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx3", VReg_96>;
650 defm SCRATCH_STORE_DWORDX4 : FLAT_Scratch_Store_Pseudo <"scratch_store_dwordx4", VReg_128>;
652 defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_byte_d16_hi", VGPR_32>;
653 defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Scratch_Store_Pseudo <"scratch_store_short_d16_hi", VGPR_32>;
655 } // End SubtargetPredicate = HasFlatScratchInsts
657 //===----------------------------------------------------------------------===//
659 //===----------------------------------------------------------------------===//
661 // Patterns for global loads with no offset.
662 class FlatLoadPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
663 (vt (node (FLATOffset i64:$vaddr, i16:$offset, i1:$slc))),
664 (inst $vaddr, $offset, 0, $slc)
667 multiclass FlatLoadPat_Hi16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt = i16> {
669 (build_vector vt:$elt0, (vt (node (FLATOffset i64:$vaddr, i16:$offset, i1:$slc)))),
670 (v2i16 (inst $vaddr, $offset, 0, $slc, $elt0))
674 (build_vector f16:$elt0, (f16 (bitconvert (vt (node (FLATOffset i64:$vaddr, i16:$offset, i1:$slc)))))),
675 (v2f16 (inst $vaddr, $offset, 0, $slc, $elt0))
679 multiclass FlatSignedLoadPat_Hi16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt = i16> {
681 (build_vector vt:$elt0, (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset, i1:$slc)))),
682 (v2i16 (inst $vaddr, $offset, 0, $slc, $elt0))
686 (build_vector f16:$elt0, (f16 (bitconvert (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset, i1:$slc)))))),
687 (v2f16 (inst $vaddr, $offset, 0, $slc, $elt0))
691 multiclass FlatLoadPat_Lo16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt = i16> {
693 (build_vector (vt (node (FLATOffset i64:$vaddr, i16:$offset, i1:$slc))), (vt (Hi16Elt vt:$hi))),
694 (v2i16 (inst $vaddr, $offset, 0, $slc, $hi))
698 (build_vector (f16 (bitconvert (vt (node (FLATOffset i64:$vaddr, i16:$offset, i1:$slc))))), (f16 (Hi16Elt f16:$hi))),
699 (v2f16 (inst $vaddr, $offset, 0, $slc, $hi))
703 multiclass FlatSignedLoadPat_Lo16 <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt = i16> {
705 (build_vector (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset, i1:$slc))), (vt (Hi16Elt vt:$hi))),
706 (v2i16 (inst $vaddr, $offset, 0, $slc, $hi))
710 (build_vector (f16 (bitconvert (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset, i1:$slc))))), (f16 (Hi16Elt f16:$hi))),
711 (v2f16 (inst $vaddr, $offset, 0, $slc, $hi))
715 class FlatLoadAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
716 (vt (node (FLATAtomic i64:$vaddr, i16:$offset, i1:$slc))),
717 (inst $vaddr, $offset, 0, $slc)
720 class FlatLoadSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
721 (vt (node (FLATOffsetSigned i64:$vaddr, i16:$offset, i1:$slc))),
722 (inst $vaddr, $offset, 0, $slc)
725 class FlatStorePat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
726 (node vt:$data, (FLATOffset i64:$vaddr, i16:$offset, i1:$slc)),
727 (inst $vaddr, $data, $offset, 0, $slc)
730 class FlatStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
731 (node vt:$data, (FLATOffsetSigned i64:$vaddr, i16:$offset, i1:$slc)),
732 (inst $vaddr, $data, $offset, 0, $slc)
735 class FlatStoreAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
736 // atomic store follows atomic binop convention so the address comes
738 (node (FLATAtomic i64:$vaddr, i16:$offset, i1:$slc), vt:$data),
739 (inst $vaddr, $data, $offset, 0, $slc)
742 class FlatStoreSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
743 // atomic store follows atomic binop convention so the address comes
745 (node (FLATSignedAtomic i64:$vaddr, i16:$offset, i1:$slc), vt:$data),
746 (inst $vaddr, $data, $offset, 0, $slc)
749 class FlatAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt,
750 ValueType data_vt = vt> : GCNPat <
751 (vt (node (FLATAtomic i64:$vaddr, i16:$offset, i1:$slc), data_vt:$data)),
752 (inst $vaddr, $data, $offset, $slc)
755 class FlatSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt,
756 ValueType data_vt = vt> : GCNPat <
757 (vt (node (FLATSignedAtomic i64:$vaddr, i16:$offset, i1:$slc), data_vt:$data)),
758 (inst $vaddr, $data, $offset, $slc)
761 let OtherPredicates = [HasFlatAddressSpace] in {
763 def : FlatLoadPat <FLAT_LOAD_UBYTE, az_extloadi8_flat, i32>;
764 def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i32>;
765 def : FlatLoadPat <FLAT_LOAD_UBYTE, az_extloadi8_flat, i16>;
766 def : FlatLoadPat <FLAT_LOAD_SBYTE, sextloadi8_flat, i16>;
767 def : FlatLoadPat <FLAT_LOAD_USHORT, az_extloadi16_flat, i32>;
768 def : FlatLoadPat <FLAT_LOAD_USHORT, load_flat, i16>;
769 def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>;
770 def : FlatLoadPat <FLAT_LOAD_DWORD, load_flat, i32>;
771 def : FlatLoadPat <FLAT_LOAD_DWORDX2, load_flat, v2i32>;
772 def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, v4i32>;
774 def : FlatLoadAtomicPat <FLAT_LOAD_DWORD, atomic_load_flat, i32>;
775 def : FlatLoadAtomicPat <FLAT_LOAD_DWORDX2, atomic_load_flat, i64>;
777 def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i32>;
778 def : FlatStorePat <FLAT_STORE_SHORT, truncstorei16_flat, i32>;
779 def : FlatStorePat <FLAT_STORE_DWORD, store_flat, i32>;
780 def : FlatStorePat <FLAT_STORE_DWORDX2, store_flat, v2i32>;
781 def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, v4i32>;
783 def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_store_flat, i32>;
784 def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_store_flat, i64>;
786 def : FlatAtomicPat <FLAT_ATOMIC_ADD_RTN, atomic_add_global, i32>;
787 def : FlatAtomicPat <FLAT_ATOMIC_SUB_RTN, atomic_sub_global, i32>;
788 def : FlatAtomicPat <FLAT_ATOMIC_INC_RTN, atomic_inc_global, i32>;
789 def : FlatAtomicPat <FLAT_ATOMIC_DEC_RTN, atomic_dec_global, i32>;
790 def : FlatAtomicPat <FLAT_ATOMIC_AND_RTN, atomic_and_global, i32>;
791 def : FlatAtomicPat <FLAT_ATOMIC_SMAX_RTN, atomic_max_global, i32>;
792 def : FlatAtomicPat <FLAT_ATOMIC_UMAX_RTN, atomic_umax_global, i32>;
793 def : FlatAtomicPat <FLAT_ATOMIC_SMIN_RTN, atomic_min_global, i32>;
794 def : FlatAtomicPat <FLAT_ATOMIC_UMIN_RTN, atomic_umin_global, i32>;
795 def : FlatAtomicPat <FLAT_ATOMIC_OR_RTN, atomic_or_global, i32>;
796 def : FlatAtomicPat <FLAT_ATOMIC_SWAP_RTN, atomic_swap_global, i32>;
797 def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_RTN, AMDGPUatomic_cmp_swap_global, i32, v2i32>;
798 def : FlatAtomicPat <FLAT_ATOMIC_XOR_RTN, atomic_xor_global, i32>;
800 def : FlatAtomicPat <FLAT_ATOMIC_ADD_X2_RTN, atomic_add_global, i64>;
801 def : FlatAtomicPat <FLAT_ATOMIC_SUB_X2_RTN, atomic_sub_global, i64>;
802 def : FlatAtomicPat <FLAT_ATOMIC_INC_X2_RTN, atomic_inc_global, i64>;
803 def : FlatAtomicPat <FLAT_ATOMIC_DEC_X2_RTN, atomic_dec_global, i64>;
804 def : FlatAtomicPat <FLAT_ATOMIC_AND_X2_RTN, atomic_and_global, i64>;
805 def : FlatAtomicPat <FLAT_ATOMIC_SMAX_X2_RTN, atomic_max_global, i64>;
806 def : FlatAtomicPat <FLAT_ATOMIC_UMAX_X2_RTN, atomic_umax_global, i64>;
807 def : FlatAtomicPat <FLAT_ATOMIC_SMIN_X2_RTN, atomic_min_global, i64>;
808 def : FlatAtomicPat <FLAT_ATOMIC_UMIN_X2_RTN, atomic_umin_global, i64>;
809 def : FlatAtomicPat <FLAT_ATOMIC_OR_X2_RTN, atomic_or_global, i64>;
810 def : FlatAtomicPat <FLAT_ATOMIC_SWAP_X2_RTN, atomic_swap_global, i64>;
811 def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global, i64, v2i64>;
812 def : FlatAtomicPat <FLAT_ATOMIC_XOR_X2_RTN, atomic_xor_global, i64>;
814 def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>;
815 def : FlatStorePat <FLAT_STORE_SHORT, store_flat, i16>;
817 let OtherPredicates = [D16PreservesUnusedBits] in {
818 def : FlatStorePat <FLAT_STORE_SHORT_D16_HI, truncstorei16_hi16_flat, i32>;
819 def : FlatStorePat <FLAT_STORE_BYTE_D16_HI, truncstorei8_hi16_flat, i32>;
821 let AddedComplexity = 3 in {
822 defm : FlatLoadPat_Hi16 <FLAT_LOAD_UBYTE_D16_HI, az_extloadi8_flat>;
823 defm : FlatLoadPat_Hi16 <FLAT_LOAD_SBYTE_D16_HI, sextloadi8_flat>;
824 defm : FlatLoadPat_Hi16 <FLAT_LOAD_SHORT_D16_HI, load_flat>;
827 let AddedComplexity = 9 in {
828 defm : FlatLoadPat_Lo16 <FLAT_LOAD_UBYTE_D16, az_extloadi8_flat>;
829 defm : FlatLoadPat_Lo16 <FLAT_LOAD_SBYTE_D16, sextloadi8_flat>;
830 defm : FlatLoadPat_Lo16 <FLAT_LOAD_SHORT_D16, load_flat>;
834 } // End OtherPredicates = [HasFlatAddressSpace]
836 let OtherPredicates = [HasFlatGlobalInsts], AddedComplexity = 10 in {
838 def : FlatLoadSignedPat <GLOBAL_LOAD_UBYTE, az_extloadi8_global, i32>;
839 def : FlatLoadSignedPat <GLOBAL_LOAD_SBYTE, sextloadi8_global, i32>;
840 def : FlatLoadSignedPat <GLOBAL_LOAD_UBYTE, az_extloadi8_global, i16>;
841 def : FlatLoadSignedPat <GLOBAL_LOAD_SBYTE, sextloadi8_global, i16>;
842 def : FlatLoadSignedPat <GLOBAL_LOAD_USHORT, az_extloadi16_global, i32>;
843 def : FlatLoadSignedPat <GLOBAL_LOAD_SSHORT, sextloadi16_global, i32>;
844 def : FlatLoadSignedPat <GLOBAL_LOAD_USHORT, load_global, i16>;
846 def : FlatLoadSignedPat <GLOBAL_LOAD_DWORD, load_global, i32>;
847 def : FlatLoadSignedPat <GLOBAL_LOAD_DWORDX2, load_global, v2i32>;
848 def : FlatLoadSignedPat <GLOBAL_LOAD_DWORDX4, load_global, v4i32>;
850 def : FlatLoadAtomicPat <GLOBAL_LOAD_DWORD, atomic_load_global, i32>;
851 def : FlatLoadAtomicPat <GLOBAL_LOAD_DWORDX2, atomic_load_global, i64>;
853 def : FlatStoreSignedPat <GLOBAL_STORE_BYTE, truncstorei8_global, i32>;
854 def : FlatStoreSignedPat <GLOBAL_STORE_BYTE, truncstorei8_global, i16>;
855 def : FlatStoreSignedPat <GLOBAL_STORE_SHORT, truncstorei16_global, i32>;
856 def : FlatStoreSignedPat <GLOBAL_STORE_SHORT, store_global, i16>;
857 def : FlatStoreSignedPat <GLOBAL_STORE_DWORD, store_global, i32>;
858 def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX2, store_global, v2i32>;
859 def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX4, store_global, v4i32>;
861 let OtherPredicates = [D16PreservesUnusedBits] in {
862 def : FlatStoreSignedPat <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>;
863 def : FlatStoreSignedPat <GLOBAL_STORE_BYTE_D16_HI, truncstorei8_hi16_global, i32>;
865 defm : FlatSignedLoadPat_Hi16 <GLOBAL_LOAD_UBYTE_D16_HI, az_extloadi8_global>;
866 defm : FlatSignedLoadPat_Hi16 <GLOBAL_LOAD_SBYTE_D16_HI, sextloadi8_global>;
867 defm : FlatSignedLoadPat_Hi16 <GLOBAL_LOAD_SHORT_D16_HI, load_global>;
869 defm : FlatSignedLoadPat_Lo16 <GLOBAL_LOAD_UBYTE_D16, az_extloadi8_global>;
870 defm : FlatSignedLoadPat_Lo16 <GLOBAL_LOAD_SBYTE_D16, sextloadi8_global>;
871 defm : FlatSignedLoadPat_Lo16 <GLOBAL_LOAD_SHORT_D16, load_global>;
875 def : FlatStoreSignedAtomicPat <GLOBAL_STORE_DWORD, store_atomic_global, i32>;
876 def : FlatStoreSignedAtomicPat <GLOBAL_STORE_DWORDX2, store_atomic_global, i64>;
878 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_ADD_RTN, atomic_add_global, i32>;
879 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SUB_RTN, atomic_sub_global, i32>;
880 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_INC_RTN, atomic_inc_global, i32>;
881 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_DEC_RTN, atomic_dec_global, i32>;
882 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_AND_RTN, atomic_and_global, i32>;
883 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SMAX_RTN, atomic_max_global, i32>;
884 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_UMAX_RTN, atomic_umax_global, i32>;
885 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SMIN_RTN, atomic_min_global, i32>;
886 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_UMIN_RTN, atomic_umin_global, i32>;
887 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_OR_RTN, atomic_or_global, i32>;
888 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SWAP_RTN, atomic_swap_global, i32>;
889 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_CMPSWAP_RTN, AMDGPUatomic_cmp_swap_global, i32, v2i32>;
890 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_XOR_RTN, atomic_xor_global, i32>;
892 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_ADD_X2_RTN, atomic_add_global, i64>;
893 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SUB_X2_RTN, atomic_sub_global, i64>;
894 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_INC_X2_RTN, atomic_inc_global, i64>;
895 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_DEC_X2_RTN, atomic_dec_global, i64>;
896 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_AND_X2_RTN, atomic_and_global, i64>;
897 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SMAX_X2_RTN, atomic_max_global, i64>;
898 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_UMAX_X2_RTN, atomic_umax_global, i64>;
899 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SMIN_X2_RTN, atomic_min_global, i64>;
900 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_UMIN_X2_RTN, atomic_umin_global, i64>;
901 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_OR_X2_RTN, atomic_or_global, i64>;
902 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SWAP_X2_RTN, atomic_swap_global, i64>;
903 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global, i64, v2i64>;
904 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_XOR_X2_RTN, atomic_xor_global, i64>;
906 } // End OtherPredicates = [HasFlatGlobalInsts]
909 //===----------------------------------------------------------------------===//
911 //===----------------------------------------------------------------------===//
913 //===----------------------------------------------------------------------===//
915 //===----------------------------------------------------------------------===//
917 class FLAT_Real_ci <bits<7> op, FLAT_Pseudo ps> :
919 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.SI> {
920 let AssemblerPredicate = isCIOnly;
921 let DecoderNamespace="CI";
924 def FLAT_LOAD_UBYTE_ci : FLAT_Real_ci <0x8, FLAT_LOAD_UBYTE>;
925 def FLAT_LOAD_SBYTE_ci : FLAT_Real_ci <0x9, FLAT_LOAD_SBYTE>;
926 def FLAT_LOAD_USHORT_ci : FLAT_Real_ci <0xa, FLAT_LOAD_USHORT>;
927 def FLAT_LOAD_SSHORT_ci : FLAT_Real_ci <0xb, FLAT_LOAD_SSHORT>;
928 def FLAT_LOAD_DWORD_ci : FLAT_Real_ci <0xc, FLAT_LOAD_DWORD>;
929 def FLAT_LOAD_DWORDX2_ci : FLAT_Real_ci <0xd, FLAT_LOAD_DWORDX2>;
930 def FLAT_LOAD_DWORDX4_ci : FLAT_Real_ci <0xe, FLAT_LOAD_DWORDX4>;
931 def FLAT_LOAD_DWORDX3_ci : FLAT_Real_ci <0xf, FLAT_LOAD_DWORDX3>;
933 def FLAT_STORE_BYTE_ci : FLAT_Real_ci <0x18, FLAT_STORE_BYTE>;
934 def FLAT_STORE_SHORT_ci : FLAT_Real_ci <0x1a, FLAT_STORE_SHORT>;
935 def FLAT_STORE_DWORD_ci : FLAT_Real_ci <0x1c, FLAT_STORE_DWORD>;
936 def FLAT_STORE_DWORDX2_ci : FLAT_Real_ci <0x1d, FLAT_STORE_DWORDX2>;
937 def FLAT_STORE_DWORDX4_ci : FLAT_Real_ci <0x1e, FLAT_STORE_DWORDX4>;
938 def FLAT_STORE_DWORDX3_ci : FLAT_Real_ci <0x1f, FLAT_STORE_DWORDX3>;
940 multiclass FLAT_Real_Atomics_ci <bits<7> op, FLAT_Pseudo ps> {
941 def _ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>;
942 def _RTN_ci : FLAT_Real_ci<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>;
945 defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_ci <0x30, FLAT_ATOMIC_SWAP>;
946 defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_ci <0x31, FLAT_ATOMIC_CMPSWAP>;
947 defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_ci <0x32, FLAT_ATOMIC_ADD>;
948 defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_ci <0x33, FLAT_ATOMIC_SUB>;
949 defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_ci <0x35, FLAT_ATOMIC_SMIN>;
950 defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_ci <0x36, FLAT_ATOMIC_UMIN>;
951 defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_ci <0x37, FLAT_ATOMIC_SMAX>;
952 defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_ci <0x38, FLAT_ATOMIC_UMAX>;
953 defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_ci <0x39, FLAT_ATOMIC_AND>;
954 defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_ci <0x3a, FLAT_ATOMIC_OR>;
955 defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_ci <0x3b, FLAT_ATOMIC_XOR>;
956 defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_ci <0x3c, FLAT_ATOMIC_INC>;
957 defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_ci <0x3d, FLAT_ATOMIC_DEC>;
958 defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_ci <0x50, FLAT_ATOMIC_SWAP_X2>;
959 defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_ci <0x51, FLAT_ATOMIC_CMPSWAP_X2>;
960 defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_ci <0x52, FLAT_ATOMIC_ADD_X2>;
961 defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_ci <0x53, FLAT_ATOMIC_SUB_X2>;
962 defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_ci <0x55, FLAT_ATOMIC_SMIN_X2>;
963 defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_ci <0x56, FLAT_ATOMIC_UMIN_X2>;
964 defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_ci <0x57, FLAT_ATOMIC_SMAX_X2>;
965 defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_ci <0x58, FLAT_ATOMIC_UMAX_X2>;
966 defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_ci <0x59, FLAT_ATOMIC_AND_X2>;
967 defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_ci <0x5a, FLAT_ATOMIC_OR_X2>;
968 defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_ci <0x5b, FLAT_ATOMIC_XOR_X2>;
969 defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_ci <0x5c, FLAT_ATOMIC_INC_X2>;
970 defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_ci <0x5d, FLAT_ATOMIC_DEC_X2>;
972 // CI Only flat instructions
973 defm FLAT_ATOMIC_FCMPSWAP : FLAT_Real_Atomics_ci <0x3e, FLAT_ATOMIC_FCMPSWAP>;
974 defm FLAT_ATOMIC_FMIN : FLAT_Real_Atomics_ci <0x3f, FLAT_ATOMIC_FMIN>;
975 defm FLAT_ATOMIC_FMAX : FLAT_Real_Atomics_ci <0x40, FLAT_ATOMIC_FMAX>;
976 defm FLAT_ATOMIC_FCMPSWAP_X2 : FLAT_Real_Atomics_ci <0x5e, FLAT_ATOMIC_FCMPSWAP_X2>;
977 defm FLAT_ATOMIC_FMIN_X2 : FLAT_Real_Atomics_ci <0x5f, FLAT_ATOMIC_FMIN_X2>;
978 defm FLAT_ATOMIC_FMAX_X2 : FLAT_Real_Atomics_ci <0x60, FLAT_ATOMIC_FMAX_X2>;
981 //===----------------------------------------------------------------------===//
983 //===----------------------------------------------------------------------===//
985 class FLAT_Real_vi <bits<7> op, FLAT_Pseudo ps> :
987 SIMCInstr <ps.PseudoInstr, SIEncodingFamily.VI> {
988 let AssemblerPredicate = isVI;
989 let DecoderNamespace="VI";
992 multiclass FLAT_Real_AllAddr_vi<bits<7> op> {
993 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME)>;
994 def _SADDR_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(NAME#"_SADDR")>;
997 def FLAT_LOAD_UBYTE_vi : FLAT_Real_vi <0x10, FLAT_LOAD_UBYTE>;
998 def FLAT_LOAD_SBYTE_vi : FLAT_Real_vi <0x11, FLAT_LOAD_SBYTE>;
999 def FLAT_LOAD_USHORT_vi : FLAT_Real_vi <0x12, FLAT_LOAD_USHORT>;
1000 def FLAT_LOAD_SSHORT_vi : FLAT_Real_vi <0x13, FLAT_LOAD_SSHORT>;
1001 def FLAT_LOAD_DWORD_vi : FLAT_Real_vi <0x14, FLAT_LOAD_DWORD>;
1002 def FLAT_LOAD_DWORDX2_vi : FLAT_Real_vi <0x15, FLAT_LOAD_DWORDX2>;
1003 def FLAT_LOAD_DWORDX4_vi : FLAT_Real_vi <0x17, FLAT_LOAD_DWORDX4>;
1004 def FLAT_LOAD_DWORDX3_vi : FLAT_Real_vi <0x16, FLAT_LOAD_DWORDX3>;
1006 def FLAT_STORE_BYTE_vi : FLAT_Real_vi <0x18, FLAT_STORE_BYTE>;
1007 def FLAT_STORE_BYTE_D16_HI_vi : FLAT_Real_vi <0x19, FLAT_STORE_BYTE_D16_HI>;
1008 def FLAT_STORE_SHORT_vi : FLAT_Real_vi <0x1a, FLAT_STORE_SHORT>;
1009 def FLAT_STORE_SHORT_D16_HI_vi : FLAT_Real_vi <0x1b, FLAT_STORE_SHORT_D16_HI>;
1010 def FLAT_STORE_DWORD_vi : FLAT_Real_vi <0x1c, FLAT_STORE_DWORD>;
1011 def FLAT_STORE_DWORDX2_vi : FLAT_Real_vi <0x1d, FLAT_STORE_DWORDX2>;
1012 def FLAT_STORE_DWORDX4_vi : FLAT_Real_vi <0x1f, FLAT_STORE_DWORDX4>;
1013 def FLAT_STORE_DWORDX3_vi : FLAT_Real_vi <0x1e, FLAT_STORE_DWORDX3>;
1015 def FLAT_LOAD_UBYTE_D16_vi : FLAT_Real_vi <0x20, FLAT_LOAD_UBYTE_D16>;
1016 def FLAT_LOAD_UBYTE_D16_HI_vi : FLAT_Real_vi <0x21, FLAT_LOAD_UBYTE_D16_HI>;
1017 def FLAT_LOAD_SBYTE_D16_vi : FLAT_Real_vi <0x22, FLAT_LOAD_SBYTE_D16>;
1018 def FLAT_LOAD_SBYTE_D16_HI_vi : FLAT_Real_vi <0x23, FLAT_LOAD_SBYTE_D16_HI>;
1019 def FLAT_LOAD_SHORT_D16_vi : FLAT_Real_vi <0x24, FLAT_LOAD_SHORT_D16>;
1020 def FLAT_LOAD_SHORT_D16_HI_vi : FLAT_Real_vi <0x25, FLAT_LOAD_SHORT_D16_HI>;
1022 multiclass FLAT_Real_Atomics_vi <bits<7> op, FLAT_Pseudo ps> {
1023 def _vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr)>;
1024 def _RTN_vi : FLAT_Real_vi<op, !cast<FLAT_Pseudo>(ps.PseudoInstr # "_RTN")>;
1027 multiclass FLAT_Global_Real_Atomics_vi<bits<7> op> :
1028 FLAT_Real_AllAddr_vi<op> {
1029 def _RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_RTN")>;
1030 def _SADDR_RTN_vi : FLAT_Real_vi <op, !cast<FLAT_Pseudo>(NAME#"_SADDR_RTN")>;
1034 defm FLAT_ATOMIC_SWAP : FLAT_Real_Atomics_vi <0x40, FLAT_ATOMIC_SWAP>;
1035 defm FLAT_ATOMIC_CMPSWAP : FLAT_Real_Atomics_vi <0x41, FLAT_ATOMIC_CMPSWAP>;
1036 defm FLAT_ATOMIC_ADD : FLAT_Real_Atomics_vi <0x42, FLAT_ATOMIC_ADD>;
1037 defm FLAT_ATOMIC_SUB : FLAT_Real_Atomics_vi <0x43, FLAT_ATOMIC_SUB>;
1038 defm FLAT_ATOMIC_SMIN : FLAT_Real_Atomics_vi <0x44, FLAT_ATOMIC_SMIN>;
1039 defm FLAT_ATOMIC_UMIN : FLAT_Real_Atomics_vi <0x45, FLAT_ATOMIC_UMIN>;
1040 defm FLAT_ATOMIC_SMAX : FLAT_Real_Atomics_vi <0x46, FLAT_ATOMIC_SMAX>;
1041 defm FLAT_ATOMIC_UMAX : FLAT_Real_Atomics_vi <0x47, FLAT_ATOMIC_UMAX>;
1042 defm FLAT_ATOMIC_AND : FLAT_Real_Atomics_vi <0x48, FLAT_ATOMIC_AND>;
1043 defm FLAT_ATOMIC_OR : FLAT_Real_Atomics_vi <0x49, FLAT_ATOMIC_OR>;
1044 defm FLAT_ATOMIC_XOR : FLAT_Real_Atomics_vi <0x4a, FLAT_ATOMIC_XOR>;
1045 defm FLAT_ATOMIC_INC : FLAT_Real_Atomics_vi <0x4b, FLAT_ATOMIC_INC>;
1046 defm FLAT_ATOMIC_DEC : FLAT_Real_Atomics_vi <0x4c, FLAT_ATOMIC_DEC>;
1047 defm FLAT_ATOMIC_SWAP_X2 : FLAT_Real_Atomics_vi <0x60, FLAT_ATOMIC_SWAP_X2>;
1048 defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Real_Atomics_vi <0x61, FLAT_ATOMIC_CMPSWAP_X2>;
1049 defm FLAT_ATOMIC_ADD_X2 : FLAT_Real_Atomics_vi <0x62, FLAT_ATOMIC_ADD_X2>;
1050 defm FLAT_ATOMIC_SUB_X2 : FLAT_Real_Atomics_vi <0x63, FLAT_ATOMIC_SUB_X2>;
1051 defm FLAT_ATOMIC_SMIN_X2 : FLAT_Real_Atomics_vi <0x64, FLAT_ATOMIC_SMIN_X2>;
1052 defm FLAT_ATOMIC_UMIN_X2 : FLAT_Real_Atomics_vi <0x65, FLAT_ATOMIC_UMIN_X2>;
1053 defm FLAT_ATOMIC_SMAX_X2 : FLAT_Real_Atomics_vi <0x66, FLAT_ATOMIC_SMAX_X2>;
1054 defm FLAT_ATOMIC_UMAX_X2 : FLAT_Real_Atomics_vi <0x67, FLAT_ATOMIC_UMAX_X2>;
1055 defm FLAT_ATOMIC_AND_X2 : FLAT_Real_Atomics_vi <0x68, FLAT_ATOMIC_AND_X2>;
1056 defm FLAT_ATOMIC_OR_X2 : FLAT_Real_Atomics_vi <0x69, FLAT_ATOMIC_OR_X2>;
1057 defm FLAT_ATOMIC_XOR_X2 : FLAT_Real_Atomics_vi <0x6a, FLAT_ATOMIC_XOR_X2>;
1058 defm FLAT_ATOMIC_INC_X2 : FLAT_Real_Atomics_vi <0x6b, FLAT_ATOMIC_INC_X2>;
1059 defm FLAT_ATOMIC_DEC_X2 : FLAT_Real_Atomics_vi <0x6c, FLAT_ATOMIC_DEC_X2>;
1061 defm GLOBAL_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>;
1062 defm GLOBAL_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>;
1063 defm GLOBAL_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>;
1064 defm GLOBAL_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>;
1065 defm GLOBAL_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>;
1066 defm GLOBAL_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>;
1067 defm GLOBAL_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>;
1068 defm GLOBAL_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>;
1070 defm GLOBAL_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>;
1071 defm GLOBAL_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>;
1072 defm GLOBAL_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>;
1073 defm GLOBAL_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>;
1074 defm GLOBAL_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>;
1075 defm GLOBAL_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>;
1077 defm GLOBAL_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>;
1078 defm GLOBAL_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>;
1079 defm GLOBAL_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>;
1080 defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>;
1081 defm GLOBAL_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>;
1082 defm GLOBAL_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>;
1083 defm GLOBAL_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>;
1084 defm GLOBAL_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>;
1087 defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Real_Atomics_vi <0x40>;
1088 defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Real_Atomics_vi <0x41>;
1089 defm GLOBAL_ATOMIC_ADD : FLAT_Global_Real_Atomics_vi <0x42>;
1090 defm GLOBAL_ATOMIC_SUB : FLAT_Global_Real_Atomics_vi <0x43>;
1091 defm GLOBAL_ATOMIC_SMIN : FLAT_Global_Real_Atomics_vi <0x44>;
1092 defm GLOBAL_ATOMIC_UMIN : FLAT_Global_Real_Atomics_vi <0x45>;
1093 defm GLOBAL_ATOMIC_SMAX : FLAT_Global_Real_Atomics_vi <0x46>;
1094 defm GLOBAL_ATOMIC_UMAX : FLAT_Global_Real_Atomics_vi <0x47>;
1095 defm GLOBAL_ATOMIC_AND : FLAT_Global_Real_Atomics_vi <0x48>;
1096 defm GLOBAL_ATOMIC_OR : FLAT_Global_Real_Atomics_vi <0x49>;
1097 defm GLOBAL_ATOMIC_XOR : FLAT_Global_Real_Atomics_vi <0x4a>;
1098 defm GLOBAL_ATOMIC_INC : FLAT_Global_Real_Atomics_vi <0x4b>;
1099 defm GLOBAL_ATOMIC_DEC : FLAT_Global_Real_Atomics_vi <0x4c>;
1100 defm GLOBAL_ATOMIC_SWAP_X2 : FLAT_Global_Real_Atomics_vi <0x60>;
1101 defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Real_Atomics_vi <0x61>;
1102 defm GLOBAL_ATOMIC_ADD_X2 : FLAT_Global_Real_Atomics_vi <0x62>;
1103 defm GLOBAL_ATOMIC_SUB_X2 : FLAT_Global_Real_Atomics_vi <0x63>;
1104 defm GLOBAL_ATOMIC_SMIN_X2 : FLAT_Global_Real_Atomics_vi <0x64>;
1105 defm GLOBAL_ATOMIC_UMIN_X2 : FLAT_Global_Real_Atomics_vi <0x65>;
1106 defm GLOBAL_ATOMIC_SMAX_X2 : FLAT_Global_Real_Atomics_vi <0x66>;
1107 defm GLOBAL_ATOMIC_UMAX_X2 : FLAT_Global_Real_Atomics_vi <0x67>;
1108 defm GLOBAL_ATOMIC_AND_X2 : FLAT_Global_Real_Atomics_vi <0x68>;
1109 defm GLOBAL_ATOMIC_OR_X2 : FLAT_Global_Real_Atomics_vi <0x69>;
1110 defm GLOBAL_ATOMIC_XOR_X2 : FLAT_Global_Real_Atomics_vi <0x6a>;
1111 defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Real_Atomics_vi <0x6b>;
1112 defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Real_Atomics_vi <0x6c>;
1114 defm SCRATCH_LOAD_UBYTE : FLAT_Real_AllAddr_vi <0x10>;
1115 defm SCRATCH_LOAD_SBYTE : FLAT_Real_AllAddr_vi <0x11>;
1116 defm SCRATCH_LOAD_USHORT : FLAT_Real_AllAddr_vi <0x12>;
1117 defm SCRATCH_LOAD_SSHORT : FLAT_Real_AllAddr_vi <0x13>;
1118 defm SCRATCH_LOAD_DWORD : FLAT_Real_AllAddr_vi <0x14>;
1119 defm SCRATCH_LOAD_DWORDX2 : FLAT_Real_AllAddr_vi <0x15>;
1120 defm SCRATCH_LOAD_DWORDX3 : FLAT_Real_AllAddr_vi <0x16>;
1121 defm SCRATCH_LOAD_DWORDX4 : FLAT_Real_AllAddr_vi <0x17>;
1122 defm SCRATCH_STORE_BYTE : FLAT_Real_AllAddr_vi <0x18>;
1123 defm SCRATCH_STORE_BYTE_D16_HI : FLAT_Real_AllAddr_vi <0x19>;
1124 defm SCRATCH_LOAD_UBYTE_D16 : FLAT_Real_AllAddr_vi <0x20>;
1125 defm SCRATCH_LOAD_UBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x21>;
1126 defm SCRATCH_LOAD_SBYTE_D16 : FLAT_Real_AllAddr_vi <0x22>;
1127 defm SCRATCH_LOAD_SBYTE_D16_HI : FLAT_Real_AllAddr_vi <0x23>;
1128 defm SCRATCH_LOAD_SHORT_D16 : FLAT_Real_AllAddr_vi <0x24>;
1129 defm SCRATCH_LOAD_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x25>;
1130 defm SCRATCH_STORE_SHORT : FLAT_Real_AllAddr_vi <0x1a>;
1131 defm SCRATCH_STORE_SHORT_D16_HI : FLAT_Real_AllAddr_vi <0x1b>;
1132 defm SCRATCH_STORE_DWORD : FLAT_Real_AllAddr_vi <0x1c>;
1133 defm SCRATCH_STORE_DWORDX2 : FLAT_Real_AllAddr_vi <0x1d>;
1134 defm SCRATCH_STORE_DWORDX3 : FLAT_Real_AllAddr_vi <0x1e>;
1135 defm SCRATCH_STORE_DWORDX4 : FLAT_Real_AllAddr_vi <0x1f>;