1 //===-- SIISelLowering.h - SI DAG Lowering Interface ------------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// SI DAG Lowering interface definition
12 //===----------------------------------------------------------------------===//
14 #ifndef LLVM_LIB_TARGET_AMDGPU_SIISELLOWERING_H
15 #define LLVM_LIB_TARGET_AMDGPU_SIISELLOWERING_H
17 #include "AMDGPUISelLowering.h"
18 #include "AMDGPUArgumentUsageInfo.h"
19 #include "SIInstrInfo.h"
23 class SITargetLowering final : public AMDGPUTargetLowering {
25 const GCNSubtarget *Subtarget;
28 MVT getRegisterTypeForCallingConv(LLVMContext &Context,
30 EVT VT) const override;
31 unsigned getNumRegistersForCallingConv(LLVMContext &Context,
33 EVT VT) const override;
35 unsigned getVectorTypeBreakdownForCallingConv(
36 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
37 unsigned &NumIntermediates, MVT &RegisterVT) const override;
40 SDValue lowerKernArgParameterPtr(SelectionDAG &DAG, const SDLoc &SL,
41 SDValue Chain, uint64_t Offset) const;
42 SDValue getImplicitArgPtr(SelectionDAG &DAG, const SDLoc &SL) const;
43 SDValue lowerKernargMemParameter(SelectionDAG &DAG, EVT VT, EVT MemVT,
44 const SDLoc &SL, SDValue Chain,
45 uint64_t Offset, Align Alignment,
47 const ISD::InputArg *Arg = nullptr) const;
49 SDValue lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA,
50 const SDLoc &SL, SDValue Chain,
51 const ISD::InputArg &Arg) const;
52 SDValue getPreloadedValue(SelectionDAG &DAG,
53 const SIMachineFunctionInfo &MFI,
55 AMDGPUFunctionArgInfo::PreloadedValue) const;
57 SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op,
58 SelectionDAG &DAG) const override;
59 SDValue lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op,
60 MVT VT, unsigned Offset) const;
61 SDValue lowerImage(SDValue Op, const AMDGPU::ImageDimIntrinsicInfo *Intr,
62 SelectionDAG &DAG) const;
63 SDValue lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, SDValue Offset,
64 SDValue CachePolicy, SelectionDAG &DAG) const;
66 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
67 SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
68 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
70 // The raw.tbuffer and struct.tbuffer intrinsics have two offset args: offset
71 // (the offset that is included in bounds checking and swizzling, to be split
72 // between the instruction's voffset and immoffset fields) and soffset (the
73 // offset that is excluded from bounds checking and swizzling, to go in the
74 // instruction's soffset field). This function takes the first kind of
75 // offset and figures out how to split it between voffset and immoffset.
76 std::pair<SDValue, SDValue> splitBufferOffsets(SDValue Offset,
77 SelectionDAG &DAG) const;
79 SDValue widenLoad(LoadSDNode *Ld, DAGCombinerInfo &DCI) const;
80 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
81 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
82 SDValue lowerFastUnsafeFDIV(SDValue Op, SelectionDAG &DAG) const;
83 SDValue lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const;
84 SDValue LowerFDIV16(SDValue Op, SelectionDAG &DAG) const;
85 SDValue LowerFDIV32(SDValue Op, SelectionDAG &DAG) const;
86 SDValue LowerFDIV64(SDValue Op, SelectionDAG &DAG) const;
87 SDValue LowerFDIV(SDValue Op, SelectionDAG &DAG) const;
88 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG, bool Signed) const;
89 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
90 SDValue LowerTrig(SDValue Op, SelectionDAG &DAG) const;
91 SDValue LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
92 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
93 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
94 SDValue adjustLoadValueType(unsigned Opcode, MemSDNode *M,
95 SelectionDAG &DAG, ArrayRef<SDValue> Ops,
96 bool IsIntrinsic = false) const;
98 SDValue lowerIntrinsicLoad(MemSDNode *M, bool IsFormat, SelectionDAG &DAG,
99 ArrayRef<SDValue> Ops) const;
101 // Call DAG.getMemIntrinsicNode for a load, but first widen a dwordx3 type to
103 SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
104 ArrayRef<SDValue> Ops, EVT MemVT,
105 MachineMemOperand *MMO, SelectionDAG &DAG) const;
107 SDValue handleD16VData(SDValue VData, SelectionDAG &DAG) const;
109 /// Converts \p Op, which must be of floating point type, to the
110 /// floating point type \p VT, by either extending or truncating it.
111 SDValue getFPExtOrFPRound(SelectionDAG &DAG,
116 SDValue convertArgType(
117 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Val,
118 bool Signed, const ISD::InputArg *Arg = nullptr) const;
120 /// Custom lowering for ISD::FP_ROUND for MVT::f16.
121 SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
122 SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
123 SDValue lowerXMULO(SDValue Op, SelectionDAG &DAG) const;
125 SDValue getSegmentAperture(unsigned AS, const SDLoc &DL,
126 SelectionDAG &DAG) const;
128 SDValue lowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) const;
129 SDValue lowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
130 SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
131 SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
132 SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
133 SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
134 SDValue lowerTRAP(SDValue Op, SelectionDAG &DAG) const;
135 SDValue lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const;
137 SDNode *adjustWritemask(MachineSDNode *&N, SelectionDAG &DAG) const;
139 SDValue performUCharToFloatCombine(SDNode *N,
140 DAGCombinerInfo &DCI) const;
141 SDValue performSHLPtrCombine(SDNode *N,
144 DAGCombinerInfo &DCI) const;
146 SDValue performMemSDNodeCombine(MemSDNode *N, DAGCombinerInfo &DCI) const;
148 SDValue splitBinaryBitConstantOp(DAGCombinerInfo &DCI, const SDLoc &SL,
149 unsigned Opc, SDValue LHS,
150 const ConstantSDNode *CRHS) const;
152 SDValue performAndCombine(SDNode *N, DAGCombinerInfo &DCI) const;
153 SDValue performOrCombine(SDNode *N, DAGCombinerInfo &DCI) const;
154 SDValue performXorCombine(SDNode *N, DAGCombinerInfo &DCI) const;
155 SDValue performZeroExtendCombine(SDNode *N, DAGCombinerInfo &DCI) const;
156 SDValue performSignExtendInRegCombine(SDNode *N, DAGCombinerInfo &DCI) const;
157 SDValue performClassCombine(SDNode *N, DAGCombinerInfo &DCI) const;
158 SDValue getCanonicalConstantFP(SelectionDAG &DAG, const SDLoc &SL, EVT VT,
159 const APFloat &C) const;
160 SDValue performFCanonicalizeCombine(SDNode *N, DAGCombinerInfo &DCI) const;
162 SDValue performFPMed3ImmCombine(SelectionDAG &DAG, const SDLoc &SL,
163 SDValue Op0, SDValue Op1) const;
164 SDValue performIntMed3ImmCombine(SelectionDAG &DAG, const SDLoc &SL,
165 SDValue Op0, SDValue Op1, bool Signed) const;
166 SDValue performMinMaxCombine(SDNode *N, DAGCombinerInfo &DCI) const;
167 SDValue performFMed3Combine(SDNode *N, DAGCombinerInfo &DCI) const;
168 SDValue performCvtPkRTZCombine(SDNode *N, DAGCombinerInfo &DCI) const;
169 SDValue performExtractVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
170 SDValue performInsertVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
172 SDValue reassociateScalarOps(SDNode *N, SelectionDAG &DAG) const;
173 unsigned getFusedOpcode(const SelectionDAG &DAG,
174 const SDNode *N0, const SDNode *N1) const;
175 SDValue performAddCombine(SDNode *N, DAGCombinerInfo &DCI) const;
176 SDValue performAddCarrySubCarryCombine(SDNode *N, DAGCombinerInfo &DCI) const;
177 SDValue performSubCombine(SDNode *N, DAGCombinerInfo &DCI) const;
178 SDValue performFAddCombine(SDNode *N, DAGCombinerInfo &DCI) const;
179 SDValue performFSubCombine(SDNode *N, DAGCombinerInfo &DCI) const;
180 SDValue performFMACombine(SDNode *N, DAGCombinerInfo &DCI) const;
181 SDValue performSetCCCombine(SDNode *N, DAGCombinerInfo &DCI) const;
182 SDValue performCvtF32UByteNCombine(SDNode *N, DAGCombinerInfo &DCI) const;
183 SDValue performClampCombine(SDNode *N, DAGCombinerInfo &DCI) const;
184 SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const;
186 bool isLegalFlatAddressingMode(const AddrMode &AM) const;
187 bool isLegalMUBUFAddressingMode(const AddrMode &AM) const;
189 unsigned isCFIntrinsic(const SDNode *Intr) const;
192 /// \returns True if fixup needs to be emitted for given global value \p GV,
194 bool shouldEmitFixup(const GlobalValue *GV) const;
196 /// \returns True if GOT relocation needs to be emitted for given global value
197 /// \p GV, false otherwise.
198 bool shouldEmitGOTReloc(const GlobalValue *GV) const;
200 /// \returns True if PC-relative relocation needs to be emitted for given
201 /// global value \p GV, false otherwise.
202 bool shouldEmitPCReloc(const GlobalValue *GV) const;
204 /// \returns true if this should use a literal constant for an LDS address,
205 /// and not emit a relocation for an LDS global.
206 bool shouldUseLDSConstAddress(const GlobalValue *GV) const;
208 /// Check if EXTRACT_VECTOR_ELT/INSERT_VECTOR_ELT (<n x e>, var-idx) should be
209 /// expanded into a set of cmp/select instructions.
210 static bool shouldExpandVectorDynExt(unsigned EltSize, unsigned NumElem,
211 bool IsDivergentIdx);
214 // Analyze a combined offset from an amdgcn_buffer_ intrinsic and store the
215 // three offsets (voffset, soffset and instoffset) into the SDValue[3] array
216 // pointed to by Offsets.
217 /// \returns 0 If there is a non-constant offset or if the offset is 0.
218 /// Otherwise returns the constant offset.
219 unsigned setBufferOffsets(SDValue CombinedOffset, SelectionDAG &DAG,
220 SDValue *Offsets, Align Alignment = Align(4)) const;
222 // Handle 8 bit and 16 bit buffer loads
223 SDValue handleByteShortBufferLoads(SelectionDAG &DAG, EVT LoadVT, SDLoc DL,
224 ArrayRef<SDValue> Ops, MemSDNode *M) const;
226 // Handle 8 bit and 16 bit buffer stores
227 SDValue handleByteShortBufferStores(SelectionDAG &DAG, EVT VDataType,
228 SDLoc DL, SDValue Ops[],
232 SITargetLowering(const TargetMachine &tm, const GCNSubtarget &STI);
234 const GCNSubtarget *getSubtarget() const;
236 bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode, EVT DestVT,
237 EVT SrcVT) const override;
239 bool isShuffleMaskLegal(ArrayRef<int> /*Mask*/, EVT /*VT*/) const override;
241 bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
243 unsigned IntrinsicID) const override;
245 bool getAddrModeArguments(IntrinsicInst * /*I*/,
246 SmallVectorImpl<Value*> &/*Ops*/,
247 Type *&/*AccessTy*/) const override;
249 bool isLegalGlobalAddressingMode(const AddrMode &AM) const;
250 bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
252 Instruction *I = nullptr) const override;
254 bool canMergeStoresTo(unsigned AS, EVT MemVT,
255 const SelectionDAG &DAG) const override;
257 bool allowsMisalignedMemoryAccessesImpl(
258 unsigned Size, unsigned AS, unsigned Align,
259 MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
260 bool *IsFast = nullptr) const;
262 bool allowsMisalignedMemoryAccesses(
263 EVT VT, unsigned AS, unsigned Align,
264 MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
265 bool *IsFast = nullptr) const override;
267 EVT getOptimalMemOpType(const MemOp &Op,
268 const AttributeList &FuncAttributes) const override;
270 bool isMemOpUniform(const SDNode *N) const;
271 bool isMemOpHasNoClobberedMemOperand(const SDNode *N) const;
273 static bool isNonGlobalAddrSpace(unsigned AS) {
274 return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS ||
275 AS == AMDGPUAS::PRIVATE_ADDRESS;
278 // FIXME: Missing constant_32bit
279 static bool isFlatGlobalAddrSpace(unsigned AS) {
280 return AS == AMDGPUAS::GLOBAL_ADDRESS ||
281 AS == AMDGPUAS::FLAT_ADDRESS ||
282 AS == AMDGPUAS::CONSTANT_ADDRESS ||
283 AS > AMDGPUAS::MAX_AMDGPU_ADDRESS;
286 bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override;
287 bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override;
289 TargetLoweringBase::LegalizeTypeAction
290 getPreferredVectorAction(MVT VT) const override;
292 bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
293 Type *Ty) const override;
295 bool isTypeDesirableForOp(unsigned Op, EVT VT) const override;
297 bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
299 bool supportSplitCSR(MachineFunction *MF) const override;
300 void initializeSplitCSR(MachineBasicBlock *Entry) const override;
301 void insertCopiesSplitCSR(
302 MachineBasicBlock *Entry,
303 const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
305 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
307 const SmallVectorImpl<ISD::InputArg> &Ins,
308 const SDLoc &DL, SelectionDAG &DAG,
309 SmallVectorImpl<SDValue> &InVals) const override;
311 bool CanLowerReturn(CallingConv::ID CallConv,
312 MachineFunction &MF, bool isVarArg,
313 const SmallVectorImpl<ISD::OutputArg> &Outs,
314 LLVMContext &Context) const override;
316 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
317 const SmallVectorImpl<ISD::OutputArg> &Outs,
318 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
319 SelectionDAG &DAG) const override;
321 void passSpecialInputs(
322 CallLoweringInfo &CLI,
324 const SIMachineFunctionInfo &Info,
325 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
326 SmallVectorImpl<SDValue> &MemOpChains,
327 SDValue Chain) const;
329 SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
330 CallingConv::ID CallConv, bool isVarArg,
331 const SmallVectorImpl<ISD::InputArg> &Ins,
332 const SDLoc &DL, SelectionDAG &DAG,
333 SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
334 SDValue ThisVal) const;
336 bool mayBeEmittedAsTailCall(const CallInst *) const override;
338 bool isEligibleForTailCallOptimization(
339 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
340 const SmallVectorImpl<ISD::OutputArg> &Outs,
341 const SmallVectorImpl<SDValue> &OutVals,
342 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const;
344 SDValue LowerCall(CallLoweringInfo &CLI,
345 SmallVectorImpl<SDValue> &InVals) const override;
347 SDValue lowerDYNAMIC_STACKALLOCImpl(SDValue Op, SelectionDAG &DAG) const;
348 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
350 Register getRegisterByName(const char* RegName, LLT VT,
351 const MachineFunction &MF) const override;
353 MachineBasicBlock *splitKillBlock(MachineInstr &MI,
354 MachineBasicBlock *BB) const;
356 void bundleInstWithWaitcnt(MachineInstr &MI) const;
357 MachineBasicBlock *emitGWSMemViolTestLoop(MachineInstr &MI,
358 MachineBasicBlock *BB) const;
361 EmitInstrWithCustomInserter(MachineInstr &MI,
362 MachineBasicBlock *BB) const override;
364 bool hasBitPreservingFPLogic(EVT VT) const override;
365 bool enableAggressiveFMAFusion(EVT VT) const override;
366 EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
367 EVT VT) const override;
368 MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override;
369 bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
370 EVT VT) const override;
371 bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const override;
373 SDValue splitUnaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
374 SDValue splitBinaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
375 SDValue splitTernaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
376 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
378 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
379 SelectionDAG &DAG) const override;
381 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
382 SDNode *PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const override;
383 void AdjustInstrPostInstrSelection(MachineInstr &MI,
384 SDNode *Node) const override;
386 SDNode *legalizeTargetIndependentNode(SDNode *Node, SelectionDAG &DAG) const;
388 MachineSDNode *wrapAddr64Rsrc(SelectionDAG &DAG, const SDLoc &DL,
390 MachineSDNode *buildRSRC(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr,
391 uint32_t RsrcDword1, uint64_t RsrcDword2And3) const;
392 std::pair<unsigned, const TargetRegisterClass *>
393 getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
394 StringRef Constraint, MVT VT) const override;
395 ConstraintType getConstraintType(StringRef Constraint) const override;
396 void LowerAsmOperandForConstraint(SDValue Op,
397 std::string &Constraint,
398 std::vector<SDValue> &Ops,
399 SelectionDAG &DAG) const override;
400 bool getAsmOperandConstVal(SDValue Op, uint64_t &Val) const;
401 bool checkAsmConstraintVal(SDValue Op,
402 const std::string &Constraint,
404 bool checkAsmConstraintValA(SDValue Op,
406 unsigned MaxSize = 64) const;
407 SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, const SDLoc &DL,
410 void finalizeLowering(MachineFunction &MF) const override;
412 void computeKnownBitsForFrameIndex(int FrameIdx,
414 const MachineFunction &MF) const override;
416 Align computeKnownAlignForTargetInstr(GISelKnownBits &Analysis, Register R,
417 const MachineRegisterInfo &MRI,
418 unsigned Depth = 0) const override;
419 bool isSDNodeSourceOfDivergence(const SDNode *N,
420 FunctionLoweringInfo *FLI, LegacyDivergenceAnalysis *DA) const override;
422 bool isCanonicalized(SelectionDAG &DAG, SDValue Op,
423 unsigned MaxDepth = 5) const;
424 bool denormalsEnabledForType(const SelectionDAG &DAG, EVT VT) const;
426 bool isKnownNeverNaNForTargetNode(SDValue Op,
427 const SelectionDAG &DAG,
429 unsigned Depth = 0) const override;
430 AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;
432 virtual const TargetRegisterClass *
433 getRegClassFor(MVT VT, bool isDivergent) const override;
434 virtual bool requiresUniformRegister(MachineFunction &MF,
435 const Value *V) const override;
436 Align getPrefLoopAlignment(MachineLoop *ML) const override;
438 void allocateHSAUserSGPRs(CCState &CCInfo,
440 const SIRegisterInfo &TRI,
441 SIMachineFunctionInfo &Info) const;
443 void allocateSystemSGPRs(CCState &CCInfo,
445 SIMachineFunctionInfo &Info,
446 CallingConv::ID CallConv,
447 bool IsShader) const;
449 void allocateSpecialEntryInputVGPRs(CCState &CCInfo,
451 const SIRegisterInfo &TRI,
452 SIMachineFunctionInfo &Info) const;
453 void allocateSpecialInputSGPRs(
456 const SIRegisterInfo &TRI,
457 SIMachineFunctionInfo &Info) const;
459 void allocateSpecialInputVGPRs(CCState &CCInfo,
461 const SIRegisterInfo &TRI,
462 SIMachineFunctionInfo &Info) const;
463 void allocateSpecialInputVGPRsFixed(CCState &CCInfo,
465 const SIRegisterInfo &TRI,
466 SIMachineFunctionInfo &Info) const;
468 std::pair<int, MVT> getTypeLegalizationCost(const DataLayout &DL,
472 } // End namespace llvm