1 //===-- AMDGPUInstructions.td - Common instruction defs ---*- tablegen -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file contains instruction defs that are common to all hw codegen
12 //===----------------------------------------------------------------------===//
14 class AddressSpacesImpl {
23 def AddrSpaces : AddressSpacesImpl;
26 class AMDGPUInst <dag outs, dag ins, string asm = "",
27 list<dag> pattern = []> : Instruction {
28 field bit isRegisterLoad = 0;
29 field bit isRegisterStore = 0;
31 let Namespace = "AMDGPU";
32 let OutOperandList = outs;
33 let InOperandList = ins;
35 let Pattern = pattern;
36 let Itinerary = NullALU;
38 // SoftFail is a field the disassembler can use to provide a way for
39 // instructions to not match without killing the whole decode process. It is
40 // mainly used for ARM, but Tablegen expects this field to exist or it fails
41 // to build the decode table.
42 field bits<64> SoftFail = 0;
44 let DecoderNamespace = Namespace;
46 let TSFlags{63} = isRegisterLoad;
47 let TSFlags{62} = isRegisterStore;
50 class AMDGPUShaderInst <dag outs, dag ins, string asm = "",
51 list<dag> pattern = []> : AMDGPUInst<outs, ins, asm, pattern> {
53 field bits<32> Inst = 0xffffffff;
56 //===---------------------------------------------------------------------===//
58 //===---------------------------------------------------------------------===//
60 class ILFormat<dag outs, dag ins, string asmstr, list<dag> pattern>
63 let Namespace = "AMDGPU";
64 dag OutOperandList = outs;
65 dag InOperandList = ins;
66 let Pattern = pattern;
67 let AsmString = !strconcat(asmstr, "\n");
69 let Itinerary = NullALU;
71 bit hasZeroOpFlag = 0;
74 let hasSideEffects = 0;
75 let isCodeGenOnly = 1;
78 def TruePredicate : Predicate<"">;
80 // Add a predicate to the list if does not already exist to deduplicate it.
81 class PredConcat<list<Predicate> lst, Predicate pred> {
83 !foldl([pred], lst, acc, cur,
84 !listconcat(acc, !if(!eq(!cast<string>(cur),!cast<string>(pred)),
88 class PredicateControl {
89 Predicate SubtargetPredicate = TruePredicate;
90 Predicate AssemblerPredicate = TruePredicate;
91 Predicate WaveSizePredicate = TruePredicate;
92 list<Predicate> OtherPredicates = [];
93 list<Predicate> Predicates = PredConcat<
94 PredConcat<PredConcat<OtherPredicates,
95 SubtargetPredicate>.ret,
96 AssemblerPredicate>.ret,
97 WaveSizePredicate>.ret;
100 class AMDGPUPat<dag pattern, dag result> : Pat<pattern, result>,
103 let RecomputePerFunction = 1 in {
104 def FP16Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().FP64FP16Denormals">;
105 def FP32Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().FP32Denormals">;
106 def FP64Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().FP64FP16Denormals">;
107 def NoFP16Denormals : Predicate<"!MF->getInfo<SIMachineFunctionInfo>()->getMode().FP64FP16Denormals">;
108 def NoFP32Denormals : Predicate<"!MF->getInfo<SIMachineFunctionInfo>()->getMode().FP32Denormals">;
109 def NoFP64Denormals : Predicate<"!MF->getInfo<SIMachineFunctionInfo>()->getMode().FP64FP16Denormals">;
110 def UnsafeFPMath : Predicate<"TM.Options.UnsafeFPMath">;
113 def FMA : Predicate<"Subtarget->hasFMA()">;
115 def InstFlag : OperandWithDefaultOps <i32, (ops (i32 0))>;
117 def u16ImmTarget : AsmOperandClass {
119 let RenderMethod = "addImmOperands";
122 def s16ImmTarget : AsmOperandClass {
124 let RenderMethod = "addImmOperands";
127 let OperandType = "OPERAND_IMMEDIATE" in {
129 def u32imm : Operand<i32> {
130 let PrintMethod = "printU32ImmOperand";
133 def u16imm : Operand<i16> {
134 let PrintMethod = "printU16ImmOperand";
135 let ParserMatchClass = u16ImmTarget;
138 def s16imm : Operand<i16> {
139 let PrintMethod = "printU16ImmOperand";
140 let ParserMatchClass = s16ImmTarget;
143 def u8imm : Operand<i8> {
144 let PrintMethod = "printU8ImmOperand";
147 } // End OperandType = "OPERAND_IMMEDIATE"
149 //===--------------------------------------------------------------------===//
151 //===--------------------------------------------------------------------===//
152 def brtarget : Operand<OtherVT>;
154 //===----------------------------------------------------------------------===//
156 //===----------------------------------------------------------------------===//
158 class HasOneUseUnaryOp<SDPatternOperator op> : PatFrag<
161 [{ return N->hasOneUse(); }]> {
163 let GISelPredicateCode = [{
164 return MRI.hasOneNonDBGUse(MI.getOperand(0).getReg());
168 class HasOneUseBinOp<SDPatternOperator op> : PatFrag<
169 (ops node:$src0, node:$src1),
171 [{ return N->hasOneUse(); }]> {
172 let GISelPredicateCode = [{
173 return MRI.hasOneNonDBGUse(MI.getOperand(0).getReg());
177 class HasOneUseTernaryOp<SDPatternOperator op> : PatFrag<
178 (ops node:$src0, node:$src1, node:$src2),
179 (op $src0, $src1, $src2),
180 [{ return N->hasOneUse(); }]> {
181 let GISelPredicateCode = [{
182 return MRI.hasOneNonDBGUse(MI.getOperand(0).getReg());
186 let Properties = [SDNPCommutative, SDNPAssociative] in {
187 def smax_oneuse : HasOneUseBinOp<smax>;
188 def smin_oneuse : HasOneUseBinOp<smin>;
189 def umax_oneuse : HasOneUseBinOp<umax>;
190 def umin_oneuse : HasOneUseBinOp<umin>;
192 def fminnum_oneuse : HasOneUseBinOp<fminnum>;
193 def fmaxnum_oneuse : HasOneUseBinOp<fmaxnum>;
195 def fminnum_ieee_oneuse : HasOneUseBinOp<fminnum_ieee>;
196 def fmaxnum_ieee_oneuse : HasOneUseBinOp<fmaxnum_ieee>;
199 def and_oneuse : HasOneUseBinOp<and>;
200 def or_oneuse : HasOneUseBinOp<or>;
201 def xor_oneuse : HasOneUseBinOp<xor>;
202 } // Properties = [SDNPCommutative, SDNPAssociative]
204 def not_oneuse : HasOneUseUnaryOp<not>;
206 def add_oneuse : HasOneUseBinOp<add>;
207 def sub_oneuse : HasOneUseBinOp<sub>;
209 def srl_oneuse : HasOneUseBinOp<srl>;
210 def shl_oneuse : HasOneUseBinOp<shl>;
212 def select_oneuse : HasOneUseTernaryOp<select>;
214 def AMDGPUmul_u24_oneuse : HasOneUseBinOp<AMDGPUmul_u24>;
215 def AMDGPUmul_i24_oneuse : HasOneUseBinOp<AMDGPUmul_i24>;
217 def srl_16 : PatFrag<
218 (ops node:$src0), (srl_oneuse node:$src0, (i32 16))
222 def hi_i16_elt : PatFrag<
223 (ops node:$src0), (i16 (trunc (i32 (srl_16 node:$src0))))
227 def hi_f16_elt : PatLeaf<
229 if (N->getOpcode() != ISD::BITCAST)
231 SDValue Tmp = N->getOperand(0);
233 if (Tmp.getOpcode() != ISD::SRL)
235 if (const auto *RHS = dyn_cast<ConstantSDNode>(Tmp.getOperand(1))
236 return RHS->getZExtValue() == 16;
240 //===----------------------------------------------------------------------===//
241 // PatLeafs for floating-point comparisons
242 //===----------------------------------------------------------------------===//
244 def COND_OEQ : PatFrags<(ops), [(OtherVT SETOEQ), (OtherVT SETEQ)]>;
245 def COND_ONE : PatFrags<(ops), [(OtherVT SETONE), (OtherVT SETNE)]>;
246 def COND_OGT : PatFrags<(ops), [(OtherVT SETOGT), (OtherVT SETGT)]>;
247 def COND_OGE : PatFrags<(ops), [(OtherVT SETOGE), (OtherVT SETGE)]>;
248 def COND_OLT : PatFrags<(ops), [(OtherVT SETOLT), (OtherVT SETLT)]>;
249 def COND_OLE : PatFrags<(ops), [(OtherVT SETOLE), (OtherVT SETLE)]>;
250 def COND_O : PatFrags<(ops), [(OtherVT SETO)]>;
251 def COND_UO : PatFrags<(ops), [(OtherVT SETUO)]>;
253 //===----------------------------------------------------------------------===//
254 // PatLeafs for unsigned / unordered comparisons
255 //===----------------------------------------------------------------------===//
257 def COND_UEQ : PatFrag<(ops), (OtherVT SETUEQ)>;
258 def COND_UNE : PatFrag<(ops), (OtherVT SETUNE)>;
259 def COND_UGT : PatFrag<(ops), (OtherVT SETUGT)>;
260 def COND_UGE : PatFrag<(ops), (OtherVT SETUGE)>;
261 def COND_ULT : PatFrag<(ops), (OtherVT SETULT)>;
262 def COND_ULE : PatFrag<(ops), (OtherVT SETULE)>;
264 // XXX - For some reason R600 version is preferring to use unordered
266 def COND_UNE_NE : PatFrags<(ops), [(OtherVT SETUNE), (OtherVT SETNE)]>;
268 //===----------------------------------------------------------------------===//
269 // PatLeafs for signed comparisons
270 //===----------------------------------------------------------------------===//
272 def COND_SGT : PatFrag<(ops), (OtherVT SETGT)>;
273 def COND_SGE : PatFrag<(ops), (OtherVT SETGE)>;
274 def COND_SLT : PatFrag<(ops), (OtherVT SETLT)>;
275 def COND_SLE : PatFrag<(ops), (OtherVT SETLE)>;
277 //===----------------------------------------------------------------------===//
278 // PatLeafs for integer equality
279 //===----------------------------------------------------------------------===//
281 def COND_EQ : PatFrags<(ops), [(OtherVT SETEQ), (OtherVT SETUEQ)]>;
282 def COND_NE : PatFrags<(ops), [(OtherVT SETNE), (OtherVT SETUNE)]>;
284 // FIXME: Should not need code predicate
285 //def COND_NULL : PatLeaf<(OtherVT null_frag)>;
286 def COND_NULL : PatLeaf <
288 [{(void)N; return false;}]
291 //===----------------------------------------------------------------------===//
292 // PatLeafs for Texture Constants
293 //===----------------------------------------------------------------------===//
295 def TEX_ARRAY : PatLeaf<
297 [{uint32_t TType = (uint32_t)N->getZExtValue();
298 return TType == 9 || TType == 10 || TType == 16;
302 def TEX_RECT : PatLeaf<
304 [{uint32_t TType = (uint32_t)N->getZExtValue();
309 def TEX_SHADOW : PatLeaf<
311 [{uint32_t TType = (uint32_t)N->getZExtValue();
312 return (TType >= 6 && TType <= 8) || TType == 13;
316 def TEX_SHADOW_ARRAY : PatLeaf<
318 [{uint32_t TType = (uint32_t)N->getZExtValue();
319 return TType == 11 || TType == 12 || TType == 17;
323 //===----------------------------------------------------------------------===//
324 // Load/Store Pattern Fragments
325 //===----------------------------------------------------------------------===//
327 def atomic_cmp_swap_glue : SDNode <"ISD::ATOMIC_CMP_SWAP", SDTAtomic3,
328 [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand, SDNPInGlue]
331 class AddressSpaceList<list<int> AS> {
332 list<int> AddrSpaces = AS;
335 class Aligned<int Bytes> {
336 int MinAlignment = Bytes;
339 class StoreHi16<SDPatternOperator op> : PatFrag <
340 (ops node:$value, node:$ptr), (op (srl node:$value, (i32 16)), node:$ptr)> {
344 def LoadAddress_constant : AddressSpaceList<[ AddrSpaces.Constant ]>;
345 def LoadAddress_global : AddressSpaceList<[ AddrSpaces.Global, AddrSpaces.Constant ]>;
346 def StoreAddress_global : AddressSpaceList<[ AddrSpaces.Global ]>;
348 def LoadAddress_flat : AddressSpaceList<[ AddrSpaces.Flat,
350 AddrSpaces.Constant ]>;
351 def StoreAddress_flat : AddressSpaceList<[ AddrSpaces.Flat, AddrSpaces.Global ]>;
353 def LoadAddress_private : AddressSpaceList<[ AddrSpaces.Private ]>;
354 def StoreAddress_private : AddressSpaceList<[ AddrSpaces.Private ]>;
356 def LoadAddress_local : AddressSpaceList<[ AddrSpaces.Local ]>;
357 def StoreAddress_local : AddressSpaceList<[ AddrSpaces.Local ]>;
359 def LoadAddress_region : AddressSpaceList<[ AddrSpaces.Region ]>;
360 def StoreAddress_region : AddressSpaceList<[ AddrSpaces.Region ]>;
364 foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in {
365 let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
367 def load_#as : PatFrag<(ops node:$ptr), (unindexedload node:$ptr)> {
369 let IsNonExtLoad = 1;
372 def extloadi8_#as : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
377 def extloadi16_#as : PatFrag<(ops node:$ptr), (extload node:$ptr)> {
382 def sextloadi8_#as : PatFrag<(ops node:$ptr), (sextload node:$ptr)> {
387 def sextloadi16_#as : PatFrag<(ops node:$ptr), (sextload node:$ptr)> {
392 def zextloadi8_#as : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
397 def zextloadi16_#as : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
402 def atomic_load_32_#as : PatFrag<(ops node:$ptr), (atomic_load_32 node:$ptr)> {
407 def atomic_load_64_#as : PatFrag<(ops node:$ptr), (atomic_load_64 node:$ptr)> {
412 def store_#as : PatFrag<(ops node:$val, node:$ptr),
413 (unindexedstore node:$val, node:$ptr)> {
415 let IsTruncStore = 0;
418 // truncstore fragments.
419 def truncstore_#as : PatFrag<(ops node:$val, node:$ptr),
420 (unindexedstore node:$val, node:$ptr)> {
422 let IsTruncStore = 1;
425 // TODO: We don't really need the truncstore here. We can use
426 // unindexedstore with MemoryVT directly, which will save an
427 // unnecessary check that the memory size is less than the value type
428 // in the generated matcher table.
429 def truncstorei8_#as : PatFrag<(ops node:$val, node:$ptr),
430 (truncstore node:$val, node:$ptr)> {
435 def truncstorei16_#as : PatFrag<(ops node:$val, node:$ptr),
436 (truncstore node:$val, node:$ptr)> {
441 def store_hi16_#as : StoreHi16 <truncstorei16>;
442 def truncstorei8_hi16_#as : StoreHi16<truncstorei8>;
443 def truncstorei16_hi16_#as : StoreHi16<truncstorei16>;
445 defm atomic_store_#as : binary_atomic_op<atomic_store>;
447 } // End let AddressSpaces = ...
448 } // End foreach AddrSpace
451 multiclass ret_noret_binary_atomic_op<SDNode atomic_op, bit IsInt = 1> {
452 foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in {
453 let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
454 defm "_"#as : binary_atomic_op<atomic_op, IsInt>;
456 let PredicateCode = [{return (SDValue(N, 0).use_empty());}] in {
457 defm "_"#as#"_noret" : binary_atomic_op<atomic_op, IsInt>;
460 let PredicateCode = [{return !(SDValue(N, 0).use_empty());}] in {
461 defm "_"#as#"_ret" : binary_atomic_op<atomic_op, IsInt>;
467 defm atomic_swap : ret_noret_binary_atomic_op<atomic_swap>;
468 defm atomic_load_add : ret_noret_binary_atomic_op<atomic_load_add>;
469 defm atomic_load_and : ret_noret_binary_atomic_op<atomic_load_and>;
470 defm atomic_load_max : ret_noret_binary_atomic_op<atomic_load_max>;
471 defm atomic_load_min : ret_noret_binary_atomic_op<atomic_load_min>;
472 defm atomic_load_or : ret_noret_binary_atomic_op<atomic_load_or>;
473 defm atomic_load_sub : ret_noret_binary_atomic_op<atomic_load_sub>;
474 defm atomic_load_umax : ret_noret_binary_atomic_op<atomic_load_umax>;
475 defm atomic_load_umin : ret_noret_binary_atomic_op<atomic_load_umin>;
476 defm atomic_load_xor : ret_noret_binary_atomic_op<atomic_load_xor>;
477 defm atomic_load_fadd : ret_noret_binary_atomic_op<atomic_load_fadd, 0>;
478 defm AMDGPUatomic_cmp_swap : ret_noret_binary_atomic_op<AMDGPUatomic_cmp_swap>;
481 def load_align8_local : PatFrag <(ops node:$ptr), (load_local node:$ptr)> {
483 let IsNonExtLoad = 1;
484 let MinAlignment = 8;
487 def load_align16_local : PatFrag <(ops node:$ptr), (load_local node:$ptr)> {
489 let IsNonExtLoad = 1;
490 let MinAlignment = 16;
493 def store_align8_local: PatFrag<(ops node:$val, node:$ptr),
494 (store_local node:$val, node:$ptr)>, Aligned<8> {
496 let IsTruncStore = 0;
499 def store_align16_local: PatFrag<(ops node:$val, node:$ptr),
500 (store_local node:$val, node:$ptr)>, Aligned<16> {
502 let IsTruncStore = 0;
505 let AddressSpaces = StoreAddress_local.AddrSpaces in {
506 defm atomic_cmp_swap_local : ternary_atomic_op<atomic_cmp_swap>;
507 defm atomic_cmp_swap_local_m0 : ternary_atomic_op<atomic_cmp_swap_glue>;
510 let AddressSpaces = StoreAddress_region.AddrSpaces in {
511 defm atomic_cmp_swap_region : ternary_atomic_op<atomic_cmp_swap>;
512 defm atomic_cmp_swap_region_m0 : ternary_atomic_op<atomic_cmp_swap_glue>;
515 //===----------------------------------------------------------------------===//
516 // Misc Pattern Fragments
517 //===----------------------------------------------------------------------===//
520 int TWO_PI = 0x40c90fdb;
522 int TWO_PI_INV = 0x3e22f983;
523 int FP_UINT_MAX_PLUS_1 = 0x4f800000; // 1 << 32 in floating point encoding
524 int FP16_ONE = 0x3C00;
525 int FP16_NEG_ONE = 0xBC00;
526 int FP32_ONE = 0x3f800000;
527 int FP32_NEG_ONE = 0xbf800000;
528 int FP64_ONE = 0x3ff0000000000000;
529 int FP64_NEG_ONE = 0xbff0000000000000;
531 def CONST : Constants;
533 def FP_ZERO : PatLeaf <
535 [{return N->getValueAPF().isZero();}]
538 def FP_ONE : PatLeaf <
540 [{return N->isExactlyValue(1.0);}]
543 def FP_HALF : PatLeaf <
545 [{return N->isExactlyValue(0.5);}]
548 /* Generic helper patterns for intrinsics */
549 /* -------------------------------------- */
551 class POW_Common <AMDGPUInst log_ieee, AMDGPUInst exp_ieee, AMDGPUInst mul>
553 (fpow f32:$src0, f32:$src1),
554 (exp_ieee (mul f32:$src1, (log_ieee f32:$src0)))
557 /* Other helper patterns */
558 /* --------------------- */
560 /* Extract element pattern */
561 class Extract_Element <ValueType sub_type, ValueType vec_type, int sub_idx,
564 (sub_type (extractelt vec_type:$src, sub_idx)),
565 (EXTRACT_SUBREG $src, sub_reg)
568 /* Insert element pattern */
569 class Insert_Element <ValueType elem_type, ValueType vec_type,
570 int sub_idx, SubRegIndex sub_reg>
572 (insertelt vec_type:$vec, elem_type:$elem, sub_idx),
573 (INSERT_SUBREG $vec, $elem, sub_reg)
576 // XXX: Convert to new syntax and use COPY_TO_REG, once the DFAPacketizer
577 // can handle COPY instructions.
578 // bitconvert pattern
579 class BitConvert <ValueType dt, ValueType st, RegisterClass rc> : AMDGPUPat <
580 (dt (bitconvert (st rc:$src0))),
584 // XXX: Convert to new syntax and use COPY_TO_REG, once the DFAPacketizer
585 // can handle COPY instructions.
586 class DwordAddrPat<ValueType vt, RegisterClass rc> : AMDGPUPat <
587 (vt (AMDGPUdwordaddr (vt rc:$addr))),
593 multiclass BFIPatterns <Instruction BFI_INT,
594 Instruction LoadImm32,
595 RegisterClass RC64> {
596 // Definition from ISA doc:
597 // (y & x) | (z & ~x)
599 (or (and i32:$y, i32:$x), (and i32:$z, (not i32:$x))),
605 (or (and i64:$y, i64:$x), (and i64:$z, (not i64:$x))),
607 (BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub0)),
608 (i32 (EXTRACT_SUBREG RC64:$y, sub0)),
609 (i32 (EXTRACT_SUBREG RC64:$z, sub0))), sub0,
610 (BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub1)),
611 (i32 (EXTRACT_SUBREG RC64:$y, sub1)),
612 (i32 (EXTRACT_SUBREG RC64:$z, sub1))), sub1)
615 // SHA-256 Ch function
618 (xor i32:$z, (and i32:$x, (xor i32:$y, i32:$z))),
624 (xor i64:$z, (and i64:$x, (xor i64:$y, i64:$z))),
626 (BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub0)),
627 (i32 (EXTRACT_SUBREG RC64:$y, sub0)),
628 (i32 (EXTRACT_SUBREG RC64:$z, sub0))), sub0,
629 (BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub1)),
630 (i32 (EXTRACT_SUBREG RC64:$y, sub1)),
631 (i32 (EXTRACT_SUBREG RC64:$z, sub1))), sub1)
635 (fcopysign f32:$src0, f32:$src1),
636 (BFI_INT (LoadImm32 (i32 0x7fffffff)), $src0, $src1)
640 (f32 (fcopysign f32:$src0, f64:$src1)),
641 (BFI_INT (LoadImm32 (i32 0x7fffffff)), $src0,
642 (i32 (EXTRACT_SUBREG RC64:$src1, sub1)))
646 (f64 (fcopysign f64:$src0, f64:$src1)),
648 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,
649 (BFI_INT (LoadImm32 (i32 0x7fffffff)),
650 (i32 (EXTRACT_SUBREG RC64:$src0, sub1)),
651 (i32 (EXTRACT_SUBREG RC64:$src1, sub1))), sub1)
655 (f64 (fcopysign f64:$src0, f32:$src1)),
657 (i32 (EXTRACT_SUBREG $src0, sub0)), sub0,
658 (BFI_INT (LoadImm32 (i32 0x7fffffff)),
659 (i32 (EXTRACT_SUBREG RC64:$src0, sub1)),
664 // SHA-256 Ma patterns
666 // ((x & z) | (y & (x | z))) -> BFI_INT (XOR x, y), z, y
667 multiclass SHA256MaPattern <Instruction BFI_INT, Instruction XOR, RegisterClass RC64> {
669 (or (and i32:$x, i32:$z), (and i32:$y, (or i32:$x, i32:$z))),
670 (BFI_INT (XOR i32:$x, i32:$y), i32:$z, i32:$y)
674 (or (and i64:$x, i64:$z), (and i64:$y, (or i64:$x, i64:$z))),
676 (BFI_INT (XOR (i32 (EXTRACT_SUBREG RC64:$x, sub0)),
677 (i32 (EXTRACT_SUBREG RC64:$y, sub0))),
678 (i32 (EXTRACT_SUBREG RC64:$z, sub0)),
679 (i32 (EXTRACT_SUBREG RC64:$y, sub0))), sub0,
680 (BFI_INT (XOR (i32 (EXTRACT_SUBREG RC64:$x, sub1)),
681 (i32 (EXTRACT_SUBREG RC64:$y, sub1))),
682 (i32 (EXTRACT_SUBREG RC64:$z, sub1)),
683 (i32 (EXTRACT_SUBREG RC64:$y, sub1))), sub1)
687 // Bitfield extract patterns
689 def IMMZeroBasedBitfieldMask : ImmLeaf <i32, [{
690 return isMask_32(Imm);
693 def IMMPopCount : SDNodeXForm<imm, [{
694 return CurDAG->getTargetConstant(countPopulation(N->getZExtValue()), SDLoc(N),
698 multiclass BFEPattern <Instruction UBFE, Instruction SBFE, Instruction MOV> {
700 (i32 (and (i32 (srl i32:$src, i32:$rshift)), IMMZeroBasedBitfieldMask:$mask)),
701 (UBFE $src, $rshift, (MOV (i32 (IMMPopCount $mask))))
704 // x & ((1 << y) - 1)
706 (and i32:$src, (add_oneuse (shl_oneuse 1, i32:$width), -1)),
707 (UBFE $src, (MOV (i32 0)), $width)
712 (and i32:$src, (xor_oneuse (shl_oneuse -1, i32:$width), -1)),
713 (UBFE $src, (MOV (i32 0)), $width)
716 // x & (-1 >> (bitwidth - y))
718 (and i32:$src, (srl_oneuse -1, (sub 32, i32:$width))),
719 (UBFE $src, (MOV (i32 0)), $width)
722 // x << (bitwidth - y) >> (bitwidth - y)
724 (srl (shl_oneuse i32:$src, (sub 32, i32:$width)), (sub 32, i32:$width)),
725 (UBFE $src, (MOV (i32 0)), $width)
729 (sra (shl_oneuse i32:$src, (sub 32, i32:$width)), (sub 32, i32:$width)),
730 (SBFE $src, (MOV (i32 0)), $width)
735 class ROTRPattern <Instruction BIT_ALIGN> : AMDGPUPat <
736 (rotr i32:$src0, i32:$src1),
737 (BIT_ALIGN $src0, $src0, $src1)
740 // Special conversion patterns
742 def cvt_rpi_i32_f32 : PatFrag <
744 (fp_to_sint (ffloor (fadd $src, FP_HALF))),
745 [{ (void) N; return TM.Options.NoNaNsFPMath; }]
748 def cvt_flr_i32_f32 : PatFrag <
750 (fp_to_sint (ffloor $src)),
751 [{ (void)N; return TM.Options.NoNaNsFPMath; }]
754 let AddedComplexity = 2 in {
755 class IMad24Pat<Instruction Inst, bit HasClamp = 0> : AMDGPUPat <
756 (add (AMDGPUmul_i24 i32:$src0, i32:$src1), i32:$src2),
757 !if(HasClamp, (Inst $src0, $src1, $src2, (i1 0)),
758 (Inst $src0, $src1, $src2))
761 class UMad24Pat<Instruction Inst, bit HasClamp = 0> : AMDGPUPat <
762 (add (AMDGPUmul_u24 i32:$src0, i32:$src1), i32:$src2),
763 !if(HasClamp, (Inst $src0, $src1, $src2, (i1 0)),
764 (Inst $src0, $src1, $src2))
766 } // AddedComplexity.
768 class RcpPat<Instruction RcpInst, ValueType vt> : AMDGPUPat <
769 (fdiv FP_ONE, vt:$src),
773 class RsqPat<Instruction RsqInst, ValueType vt> : AMDGPUPat <
774 (AMDGPUrcp (fsqrt vt:$src)),
778 // Instructions which select to the same v_min_f*
779 def fminnum_like : PatFrags<(ops node:$src0, node:$src1),
780 [(fminnum_ieee node:$src0, node:$src1),
781 (fminnum node:$src0, node:$src1)]
784 // Instructions which select to the same v_max_f*
785 def fmaxnum_like : PatFrags<(ops node:$src0, node:$src1),
786 [(fmaxnum_ieee node:$src0, node:$src1),
787 (fmaxnum node:$src0, node:$src1)]
790 def fminnum_like_oneuse : PatFrags<(ops node:$src0, node:$src1),
791 [(fminnum_ieee_oneuse node:$src0, node:$src1),
792 (fminnum_oneuse node:$src0, node:$src1)]
795 def fmaxnum_like_oneuse : PatFrags<(ops node:$src0, node:$src1),
796 [(fmaxnum_ieee_oneuse node:$src0, node:$src1),
797 (fmaxnum_oneuse node:$src0, node:$src1)]