1 //===- AMDGPULegalizerInfo.cpp -----------------------------------*- C++ -*-==//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 /// This file implements the targeting of the Machinelegalizer class for
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
14 #include "AMDGPULegalizerInfo.h"
17 #include "AMDGPUGlobalISelUtils.h"
18 #include "AMDGPUTargetMachine.h"
19 #include "SIMachineFunctionInfo.h"
20 #include "llvm/ADT/ScopeExit.h"
21 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
22 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
23 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
24 #include "llvm/CodeGen/TargetOpcodes.h"
25 #include "llvm/CodeGen/ValueTypes.h"
26 #include "llvm/IR/DerivedTypes.h"
27 #include "llvm/IR/DiagnosticInfo.h"
28 #include "llvm/IR/Type.h"
29 #include "llvm/Support/Debug.h"
31 #define DEBUG_TYPE "amdgpu-legalinfo"
34 using namespace LegalizeActions;
35 using namespace LegalizeMutations;
36 using namespace LegalityPredicates;
37 using namespace MIPatternMatch;
39 // Hack until load/store selection patterns support any tuple of legal types.
40 static cl::opt<bool> EnableNewLegality(
41 "amdgpu-global-isel-new-legality",
42 cl::desc("Use GlobalISel desired legality, rather than try to use"
43 "rules compatible with selection patterns"),
47 static constexpr unsigned MaxRegisterSize = 1024;
49 // Round the number of elements to the next power of two elements
50 static LLT getPow2VectorType(LLT Ty) {
51 unsigned NElts = Ty.getNumElements();
52 unsigned Pow2NElts = 1 << Log2_32_Ceil(NElts);
53 return Ty.changeNumElements(Pow2NElts);
56 // Round the number of bits to the next power of two bits
57 static LLT getPow2ScalarType(LLT Ty) {
58 unsigned Bits = Ty.getSizeInBits();
59 unsigned Pow2Bits = 1 << Log2_32_Ceil(Bits);
60 return LLT::scalar(Pow2Bits);
63 static LegalityPredicate isSmallOddVector(unsigned TypeIdx) {
64 return [=](const LegalityQuery &Query) {
65 const LLT Ty = Query.Types[TypeIdx];
66 return Ty.isVector() &&
67 Ty.getNumElements() % 2 != 0 &&
68 Ty.getElementType().getSizeInBits() < 32 &&
69 Ty.getSizeInBits() % 32 != 0;
73 static LegalityPredicate isWideVec16(unsigned TypeIdx) {
74 return [=](const LegalityQuery &Query) {
75 const LLT Ty = Query.Types[TypeIdx];
76 const LLT EltTy = Ty.getScalarType();
77 return EltTy.getSizeInBits() == 16 && Ty.getNumElements() > 2;
81 static LegalizeMutation oneMoreElement(unsigned TypeIdx) {
82 return [=](const LegalityQuery &Query) {
83 const LLT Ty = Query.Types[TypeIdx];
84 const LLT EltTy = Ty.getElementType();
85 return std::make_pair(TypeIdx, LLT::vector(Ty.getNumElements() + 1, EltTy));
89 static LegalizeMutation fewerEltsToSize64Vector(unsigned TypeIdx) {
90 return [=](const LegalityQuery &Query) {
91 const LLT Ty = Query.Types[TypeIdx];
92 const LLT EltTy = Ty.getElementType();
93 unsigned Size = Ty.getSizeInBits();
94 unsigned Pieces = (Size + 63) / 64;
95 unsigned NewNumElts = (Ty.getNumElements() + 1) / Pieces;
96 return std::make_pair(TypeIdx, LLT::scalarOrVector(NewNumElts, EltTy));
100 // Increase the number of vector elements to reach the next multiple of 32-bit
102 static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx) {
103 return [=](const LegalityQuery &Query) {
104 const LLT Ty = Query.Types[TypeIdx];
106 const LLT EltTy = Ty.getElementType();
107 const int Size = Ty.getSizeInBits();
108 const int EltSize = EltTy.getSizeInBits();
109 const int NextMul32 = (Size + 31) / 32;
111 assert(EltSize < 32);
113 const int NewNumElts = (32 * NextMul32 + EltSize - 1) / EltSize;
114 return std::make_pair(TypeIdx, LLT::vector(NewNumElts, EltTy));
118 static LegalizeMutation bitcastToRegisterType(unsigned TypeIdx) {
119 return [=](const LegalityQuery &Query) {
120 const LLT Ty = Query.Types[TypeIdx];
121 unsigned Size = Ty.getSizeInBits();
127 CoercedTy = LLT::scalar(Size);
129 CoercedTy = LLT::scalarOrVector(Size / 32, 32);
131 return std::make_pair(TypeIdx, CoercedTy);
135 static LegalityPredicate vectorSmallerThan(unsigned TypeIdx, unsigned Size) {
136 return [=](const LegalityQuery &Query) {
137 const LLT QueryTy = Query.Types[TypeIdx];
138 return QueryTy.isVector() && QueryTy.getSizeInBits() < Size;
142 static LegalityPredicate vectorWiderThan(unsigned TypeIdx, unsigned Size) {
143 return [=](const LegalityQuery &Query) {
144 const LLT QueryTy = Query.Types[TypeIdx];
145 return QueryTy.isVector() && QueryTy.getSizeInBits() > Size;
149 static LegalityPredicate numElementsNotEven(unsigned TypeIdx) {
150 return [=](const LegalityQuery &Query) {
151 const LLT QueryTy = Query.Types[TypeIdx];
152 return QueryTy.isVector() && QueryTy.getNumElements() % 2 != 0;
156 static bool isRegisterSize(unsigned Size) {
157 return Size % 32 == 0 && Size <= MaxRegisterSize;
160 static bool isRegisterVectorElementType(LLT EltTy) {
161 const int EltSize = EltTy.getSizeInBits();
162 return EltSize == 16 || EltSize % 32 == 0;
165 static bool isRegisterVectorType(LLT Ty) {
166 const int EltSize = Ty.getElementType().getSizeInBits();
167 return EltSize == 32 || EltSize == 64 ||
168 (EltSize == 16 && Ty.getNumElements() % 2 == 0) ||
169 EltSize == 128 || EltSize == 256;
172 static bool isRegisterType(LLT Ty) {
173 if (!isRegisterSize(Ty.getSizeInBits()))
177 return isRegisterVectorType(Ty);
182 // Any combination of 32 or 64-bit elements up the maximum register size, and
183 // multiples of v2s16.
184 static LegalityPredicate isRegisterType(unsigned TypeIdx) {
185 return [=](const LegalityQuery &Query) {
186 return isRegisterType(Query.Types[TypeIdx]);
190 static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx) {
191 return [=](const LegalityQuery &Query) {
192 const LLT QueryTy = Query.Types[TypeIdx];
193 if (!QueryTy.isVector())
195 const LLT EltTy = QueryTy.getElementType();
196 return EltTy == LLT::scalar(16) || EltTy.getSizeInBits() >= 32;
200 static LegalityPredicate isWideScalarTruncStore(unsigned TypeIdx) {
201 return [=](const LegalityQuery &Query) {
202 const LLT Ty = Query.Types[TypeIdx];
203 return !Ty.isVector() && Ty.getSizeInBits() > 32 &&
204 Query.MMODescrs[0].SizeInBits < Ty.getSizeInBits();
208 // TODO: Should load to s16 be legal? Most loads extend to 32-bits, but we
209 // handle some operations by just promoting the register during
210 // selection. There are also d16 loads on GFX9+ which preserve the high bits.
211 static unsigned maxSizeForAddrSpace(const GCNSubtarget &ST, unsigned AS,
214 case AMDGPUAS::PRIVATE_ADDRESS:
215 // FIXME: Private element size.
217 case AMDGPUAS::LOCAL_ADDRESS:
218 return ST.useDS128() ? 128 : 64;
219 case AMDGPUAS::GLOBAL_ADDRESS:
220 case AMDGPUAS::CONSTANT_ADDRESS:
221 case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
222 // Treat constant and global as identical. SMRD loads are sometimes usable for
223 // global loads (ideally constant address space should be eliminated)
224 // depending on the context. Legality cannot be context dependent, but
225 // RegBankSelect can split the load as necessary depending on the pointer
226 // register bank/uniformity and if the memory is invariant or not written in a
228 return IsLoad ? 512 : 128;
230 // Flat addresses may contextually need to be split to 32-bit parts if they
231 // may alias scratch depending on the subtarget.
236 static bool isLoadStoreSizeLegal(const GCNSubtarget &ST,
237 const LegalityQuery &Query,
239 const LLT Ty = Query.Types[0];
241 // Handle G_LOAD, G_ZEXTLOAD, G_SEXTLOAD
242 const bool IsLoad = Opcode != AMDGPU::G_STORE;
244 unsigned RegSize = Ty.getSizeInBits();
245 unsigned MemSize = Query.MMODescrs[0].SizeInBits;
246 unsigned Align = Query.MMODescrs[0].AlignInBits;
247 unsigned AS = Query.Types[1].getAddressSpace();
249 // All of these need to be custom lowered to cast the pointer operand.
250 if (AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
253 // TODO: We should be able to widen loads if the alignment is high enough, but
254 // we also need to modify the memory access size.
256 // Accept widening loads based on alignment.
257 if (IsLoad && MemSize < Size)
258 MemSize = std::max(MemSize, Align);
261 // Only 1-byte and 2-byte to 32-bit extloads are valid.
262 if (MemSize != RegSize && RegSize != 32)
265 if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad))
276 if (!ST.hasDwordx3LoadStores())
281 // These may contextually need to be broken down.
287 assert(RegSize >= MemSize);
289 if (Align < MemSize) {
290 const SITargetLowering *TLI = ST.getTargetLowering();
291 if (!TLI->allowsMisalignedMemoryAccessesImpl(MemSize, AS, Align / 8))
298 // The current selector can't handle <6 x s16>, <8 x s16>, s96, s128 etc, so
299 // workaround this. Eventually it should ignore the type for loads and only care
300 // about the size. Return true in cases where we will workaround this for now by
302 static bool loadStoreBitcastWorkaround(const LLT Ty) {
303 if (EnableNewLegality)
306 const unsigned Size = Ty.getSizeInBits();
311 unsigned EltSize = Ty.getElementType().getSizeInBits();
312 return EltSize != 32 && EltSize != 64;
315 static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query,
317 const LLT Ty = Query.Types[0];
318 return isRegisterType(Ty) && isLoadStoreSizeLegal(ST, Query, Opcode) &&
319 !loadStoreBitcastWorkaround(Ty);
322 AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
323 const GCNTargetMachine &TM)
325 using namespace TargetOpcode;
327 auto GetAddrSpacePtr = [&TM](unsigned AS) {
328 return LLT::pointer(AS, TM.getPointerSizeInBits(AS));
331 const LLT S1 = LLT::scalar(1);
332 const LLT S16 = LLT::scalar(16);
333 const LLT S32 = LLT::scalar(32);
334 const LLT S64 = LLT::scalar(64);
335 const LLT S128 = LLT::scalar(128);
336 const LLT S256 = LLT::scalar(256);
337 const LLT S512 = LLT::scalar(512);
338 const LLT MaxScalar = LLT::scalar(MaxRegisterSize);
340 const LLT V2S16 = LLT::vector(2, 16);
341 const LLT V4S16 = LLT::vector(4, 16);
343 const LLT V2S32 = LLT::vector(2, 32);
344 const LLT V3S32 = LLT::vector(3, 32);
345 const LLT V4S32 = LLT::vector(4, 32);
346 const LLT V5S32 = LLT::vector(5, 32);
347 const LLT V6S32 = LLT::vector(6, 32);
348 const LLT V7S32 = LLT::vector(7, 32);
349 const LLT V8S32 = LLT::vector(8, 32);
350 const LLT V9S32 = LLT::vector(9, 32);
351 const LLT V10S32 = LLT::vector(10, 32);
352 const LLT V11S32 = LLT::vector(11, 32);
353 const LLT V12S32 = LLT::vector(12, 32);
354 const LLT V13S32 = LLT::vector(13, 32);
355 const LLT V14S32 = LLT::vector(14, 32);
356 const LLT V15S32 = LLT::vector(15, 32);
357 const LLT V16S32 = LLT::vector(16, 32);
358 const LLT V32S32 = LLT::vector(32, 32);
360 const LLT V2S64 = LLT::vector(2, 64);
361 const LLT V3S64 = LLT::vector(3, 64);
362 const LLT V4S64 = LLT::vector(4, 64);
363 const LLT V5S64 = LLT::vector(5, 64);
364 const LLT V6S64 = LLT::vector(6, 64);
365 const LLT V7S64 = LLT::vector(7, 64);
366 const LLT V8S64 = LLT::vector(8, 64);
367 const LLT V16S64 = LLT::vector(16, 64);
369 std::initializer_list<LLT> AllS32Vectors =
370 {V2S32, V3S32, V4S32, V5S32, V6S32, V7S32, V8S32,
371 V9S32, V10S32, V11S32, V12S32, V13S32, V14S32, V15S32, V16S32, V32S32};
372 std::initializer_list<LLT> AllS64Vectors =
373 {V2S64, V3S64, V4S64, V5S64, V6S64, V7S64, V8S64, V16S64};
375 const LLT GlobalPtr = GetAddrSpacePtr(AMDGPUAS::GLOBAL_ADDRESS);
376 const LLT ConstantPtr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS);
377 const LLT Constant32Ptr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS_32BIT);
378 const LLT LocalPtr = GetAddrSpacePtr(AMDGPUAS::LOCAL_ADDRESS);
379 const LLT RegionPtr = GetAddrSpacePtr(AMDGPUAS::REGION_ADDRESS);
380 const LLT FlatPtr = GetAddrSpacePtr(AMDGPUAS::FLAT_ADDRESS);
381 const LLT PrivatePtr = GetAddrSpacePtr(AMDGPUAS::PRIVATE_ADDRESS);
383 const LLT CodePtr = FlatPtr;
385 const std::initializer_list<LLT> AddrSpaces64 = {
386 GlobalPtr, ConstantPtr, FlatPtr
389 const std::initializer_list<LLT> AddrSpaces32 = {
390 LocalPtr, PrivatePtr, Constant32Ptr, RegionPtr
393 const std::initializer_list<LLT> FPTypesBase = {
397 const std::initializer_list<LLT> FPTypes16 = {
401 const std::initializer_list<LLT> FPTypesPK16 = {
405 const LLT MinScalarFPTy = ST.has16BitInsts() ? S16 : S32;
407 setAction({G_BRCOND, S1}, Legal); // VCC branches
408 setAction({G_BRCOND, S32}, Legal); // SCC branches
410 // TODO: All multiples of 32, vectors of pointers, all v2s16 pairs, more
411 // elements for v3s16
412 getActionDefinitionsBuilder(G_PHI)
413 .legalFor({S32, S64, V2S16, V4S16, S1, S128, S256})
414 .legalFor(AllS32Vectors)
415 .legalFor(AllS64Vectors)
416 .legalFor(AddrSpaces64)
417 .legalFor(AddrSpaces32)
418 .clampScalar(0, S32, S256)
419 .widenScalarToNextPow2(0, 32)
420 .clampMaxNumElements(0, S32, 16)
421 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
422 .legalIf(isPointer(0));
424 if (ST.hasVOP3PInsts()) {
425 getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
426 .legalFor({S32, S16, V2S16})
427 .clampScalar(0, S16, S32)
428 .clampMaxNumElements(0, S16, 2)
430 .widenScalarToNextPow2(0, 32);
431 } else if (ST.has16BitInsts()) {
432 getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
433 .legalFor({S32, S16})
434 .clampScalar(0, S16, S32)
436 .widenScalarToNextPow2(0, 32);
438 getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
440 .clampScalar(0, S32, S32)
444 // FIXME: Not really legal. Placeholder for custom lowering.
445 getActionDefinitionsBuilder({G_SDIV, G_UDIV, G_SREM, G_UREM})
446 .customFor({S32, S64})
447 .clampScalar(0, S32, S64)
448 .widenScalarToNextPow2(0, 32)
451 getActionDefinitionsBuilder({G_UMULH, G_SMULH})
453 .clampScalar(0, S32, S32)
456 // Report legal for any types we can handle anywhere. For the cases only legal
457 // on the SALU, RegBankSelect will be able to re-legalize.
458 getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})
459 .legalFor({S32, S1, S64, V2S32, S16, V2S16, V4S16})
460 .clampScalar(0, S32, S64)
461 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
462 .fewerElementsIf(vectorWiderThan(0, 64), fewerEltsToSize64Vector(0))
463 .widenScalarToNextPow2(0)
466 getActionDefinitionsBuilder({G_UADDO, G_USUBO,
467 G_UADDE, G_SADDE, G_USUBE, G_SSUBE})
468 .legalFor({{S32, S1}, {S32, S32}})
470 // TODO: .scalarize(0)
473 getActionDefinitionsBuilder(G_BITCAST)
474 // Don't worry about the size constraint.
475 .legalIf(all(isRegisterType(0), isRegisterType(1)))
479 getActionDefinitionsBuilder(G_CONSTANT)
480 .legalFor({S1, S32, S64, S16, GlobalPtr,
481 LocalPtr, ConstantPtr, PrivatePtr, FlatPtr })
482 .clampScalar(0, S32, S64)
483 .widenScalarToNextPow2(0)
484 .legalIf(isPointer(0));
486 getActionDefinitionsBuilder(G_FCONSTANT)
487 .legalFor({S32, S64, S16})
488 .clampScalar(0, S16, S64);
490 getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})
491 .legalIf(isRegisterType(0))
492 // s1 and s16 are special cases because they have legal operations on
493 // them, but don't really occupy registers in the normal way.
495 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
496 .clampScalarOrElt(0, S32, MaxScalar)
497 .widenScalarToNextPow2(0, 32)
498 .clampMaxNumElements(0, S32, 16);
500 setAction({G_FRAME_INDEX, PrivatePtr}, Legal);
502 // If the amount is divergent, we have to do a wave reduction to get the
503 // maximum value, so this is expanded during RegBankSelect.
504 getActionDefinitionsBuilder(G_DYN_STACKALLOC)
505 .legalFor({{PrivatePtr, S32}});
507 getActionDefinitionsBuilder(G_GLOBAL_VALUE)
508 .unsupportedFor({PrivatePtr})
510 setAction({G_BLOCK_ADDR, CodePtr}, Legal);
512 auto &FPOpActions = getActionDefinitionsBuilder(
513 { G_FADD, G_FMUL, G_FMA, G_FCANONICALIZE})
514 .legalFor({S32, S64});
515 auto &TrigActions = getActionDefinitionsBuilder({G_FSIN, G_FCOS})
516 .customFor({S32, S64});
517 auto &FDIVActions = getActionDefinitionsBuilder(G_FDIV)
518 .customFor({S32, S64});
520 if (ST.has16BitInsts()) {
521 if (ST.hasVOP3PInsts())
522 FPOpActions.legalFor({S16, V2S16});
524 FPOpActions.legalFor({S16});
526 TrigActions.customFor({S16});
527 FDIVActions.customFor({S16});
530 auto &MinNumMaxNum = getActionDefinitionsBuilder({
531 G_FMINNUM, G_FMAXNUM, G_FMINNUM_IEEE, G_FMAXNUM_IEEE});
533 if (ST.hasVOP3PInsts()) {
534 MinNumMaxNum.customFor(FPTypesPK16)
535 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
536 .clampMaxNumElements(0, S16, 2)
537 .clampScalar(0, S16, S64)
539 } else if (ST.has16BitInsts()) {
540 MinNumMaxNum.customFor(FPTypes16)
541 .clampScalar(0, S16, S64)
544 MinNumMaxNum.customFor(FPTypesBase)
545 .clampScalar(0, S32, S64)
549 if (ST.hasVOP3PInsts())
550 FPOpActions.clampMaxNumElements(0, S16, 2);
554 .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);
558 .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);
562 .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);
564 getActionDefinitionsBuilder({G_FNEG, G_FABS})
565 .legalFor(FPTypesPK16)
566 .clampMaxNumElements(0, S16, 2)
568 .clampScalar(0, S16, S64);
570 if (ST.has16BitInsts()) {
571 getActionDefinitionsBuilder({G_FSQRT, G_FFLOOR})
572 .legalFor({S32, S64, S16})
574 .clampScalar(0, S16, S64);
576 getActionDefinitionsBuilder(G_FSQRT)
577 .legalFor({S32, S64})
579 .clampScalar(0, S32, S64);
581 if (ST.hasFractBug()) {
582 getActionDefinitionsBuilder(G_FFLOOR)
584 .legalFor({S32, S64})
586 .clampScalar(0, S32, S64);
588 getActionDefinitionsBuilder(G_FFLOOR)
589 .legalFor({S32, S64})
591 .clampScalar(0, S32, S64);
595 getActionDefinitionsBuilder(G_FPTRUNC)
596 .legalFor({{S32, S64}, {S16, S32}})
600 getActionDefinitionsBuilder(G_FPEXT)
601 .legalFor({{S64, S32}, {S32, S16}})
602 .lowerFor({{S64, S16}}) // FIXME: Implement
605 getActionDefinitionsBuilder(G_FSUB)
606 // Use actual fsub instruction
608 // Must use fadd + fneg
609 .lowerFor({S64, S16, V2S16})
611 .clampScalar(0, S32, S64);
613 // Whether this is legal depends on the floating point mode for the function.
614 auto &FMad = getActionDefinitionsBuilder(G_FMAD);
615 if (ST.hasMadF16() && ST.hasMadMacF32Insts())
616 FMad.customFor({S32, S16});
617 else if (ST.hasMadMacF32Insts())
618 FMad.customFor({S32});
619 else if (ST.hasMadF16())
620 FMad.customFor({S16});
624 // TODO: Do we need to clamp maximum bitwidth?
625 getActionDefinitionsBuilder(G_TRUNC)
626 .legalIf(isScalar(0))
627 .legalFor({{V2S16, V2S32}})
628 .clampMaxNumElements(0, S16, 2)
629 // Avoid scalarizing in cases that should be truly illegal. In unresolvable
630 // situations (like an invalid implicit use), we don't want to infinite loop
632 .fewerElementsIf(elementTypeIsLegal(0), LegalizeMutations::scalarize(0))
635 getActionDefinitionsBuilder({G_SEXT, G_ZEXT, G_ANYEXT})
636 .legalFor({{S64, S32}, {S32, S16}, {S64, S16},
637 {S32, S1}, {S64, S1}, {S16, S1}})
639 .clampScalar(0, S32, S64)
640 .widenScalarToNextPow2(1, 32);
642 // TODO: Split s1->s64 during regbankselect for VALU.
643 auto &IToFP = getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
644 .legalFor({{S32, S32}, {S64, S32}, {S16, S32}})
645 .lowerFor({{S32, S64}})
646 .lowerIf(typeIs(1, S1))
647 .customFor({{S64, S64}});
648 if (ST.has16BitInsts())
649 IToFP.legalFor({{S16, S16}});
650 IToFP.clampScalar(1, S32, S64)
652 .widenScalarToNextPow2(1);
654 auto &FPToI = getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
655 .legalFor({{S32, S32}, {S32, S64}, {S32, S16}})
656 .customFor({{S64, S64}});
657 if (ST.has16BitInsts())
658 FPToI.legalFor({{S16, S16}});
660 FPToI.minScalar(1, S32);
662 FPToI.minScalar(0, S32)
666 getActionDefinitionsBuilder(G_INTRINSIC_ROUND)
670 if (ST.has16BitInsts()) {
671 getActionDefinitionsBuilder({G_INTRINSIC_TRUNC, G_FCEIL, G_FRINT})
672 .legalFor({S16, S32, S64})
673 .clampScalar(0, S16, S64)
675 } else if (ST.getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS) {
676 getActionDefinitionsBuilder({G_INTRINSIC_TRUNC, G_FCEIL, G_FRINT})
677 .legalFor({S32, S64})
678 .clampScalar(0, S32, S64)
681 getActionDefinitionsBuilder({G_INTRINSIC_TRUNC, G_FCEIL, G_FRINT})
684 .clampScalar(0, S32, S64)
688 // FIXME: Clamp offset operand.
689 getActionDefinitionsBuilder(G_PTR_ADD)
690 .legalIf(isPointer(0))
693 getActionDefinitionsBuilder(G_PTRMASK)
694 .legalIf(typeInSet(1, {S64, S32}))
696 .maxScalarIf(sizeIs(0, 32), 1, S32)
697 .maxScalarIf(sizeIs(0, 64), 1, S64)
701 getActionDefinitionsBuilder(G_ICMP)
702 // The compare output type differs based on the register bank of the output,
703 // so make both s1 and s32 legal.
705 // Scalar compares producing output in scc will be promoted to s32, as that
706 // is the allocatable register type that will be needed for the copy from
707 // scc. This will be promoted during RegBankSelect, and we assume something
708 // before that won't try to use s32 result types.
710 // Vector compares producing an output in vcc/SGPR will use s1 in VCC reg
712 .legalForCartesianProduct(
713 {S1}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr})
714 .legalForCartesianProduct(
715 {S32}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr});
716 if (ST.has16BitInsts()) {
717 CmpBuilder.legalFor({{S1, S16}});
721 .widenScalarToNextPow2(1)
722 .clampScalar(1, S32, S64)
724 .legalIf(all(typeInSet(0, {S1, S32}), isPointer(1)));
726 getActionDefinitionsBuilder(G_FCMP)
727 .legalForCartesianProduct({S1}, ST.has16BitInsts() ? FPTypes16 : FPTypesBase)
728 .widenScalarToNextPow2(1)
729 .clampScalar(1, S32, S64)
732 // FIXME: fpow has a selection pattern that should move to custom lowering.
733 auto &Exp2Ops = getActionDefinitionsBuilder({G_FEXP2, G_FLOG2});
734 if (ST.has16BitInsts())
735 Exp2Ops.legalFor({S32, S16});
737 Exp2Ops.legalFor({S32});
738 Exp2Ops.clampScalar(0, MinScalarFPTy, S32);
739 Exp2Ops.scalarize(0);
741 auto &ExpOps = getActionDefinitionsBuilder({G_FEXP, G_FLOG, G_FLOG10, G_FPOW});
742 if (ST.has16BitInsts())
743 ExpOps.customFor({{S32}, {S16}});
745 ExpOps.customFor({S32});
746 ExpOps.clampScalar(0, MinScalarFPTy, S32)
749 // The 64-bit versions produce 32-bit results, but only on the SALU.
750 getActionDefinitionsBuilder(G_CTPOP)
751 .legalFor({{S32, S32}, {S32, S64}})
752 .clampScalar(0, S32, S32)
753 .clampScalar(1, S32, S64)
755 .widenScalarToNextPow2(0, 32)
756 .widenScalarToNextPow2(1, 32);
758 // The hardware instructions return a different result on 0 than the generic
759 // instructions expect. The hardware produces -1, but these produce the
761 getActionDefinitionsBuilder({G_CTLZ, G_CTTZ})
763 .clampScalar(0, S32, S32)
764 .clampScalar(1, S32, S64)
765 .widenScalarToNextPow2(0, 32)
766 .widenScalarToNextPow2(1, 32)
769 // The 64-bit versions produce 32-bit results, but only on the SALU.
770 getActionDefinitionsBuilder({G_CTLZ_ZERO_UNDEF, G_CTTZ_ZERO_UNDEF})
771 .legalFor({{S32, S32}, {S32, S64}})
772 .clampScalar(0, S32, S32)
773 .clampScalar(1, S32, S64)
775 .widenScalarToNextPow2(0, 32)
776 .widenScalarToNextPow2(1, 32);
778 getActionDefinitionsBuilder(G_BITREVERSE)
780 .clampScalar(0, S32, S32)
783 if (ST.has16BitInsts()) {
784 getActionDefinitionsBuilder(G_BSWAP)
785 .legalFor({S16, S32, V2S16})
786 .clampMaxNumElements(0, S16, 2)
787 // FIXME: Fixing non-power-of-2 before clamp is workaround for
788 // narrowScalar limitation.
789 .widenScalarToNextPow2(0)
790 .clampScalar(0, S16, S32)
793 if (ST.hasVOP3PInsts()) {
794 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
795 .legalFor({S32, S16, V2S16})
796 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
797 .clampMaxNumElements(0, S16, 2)
799 .widenScalarToNextPow2(0)
803 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
804 .legalFor({S32, S16})
805 .widenScalarToNextPow2(0)
811 // TODO: Should have same legality without v_perm_b32
812 getActionDefinitionsBuilder(G_BSWAP)
814 .lowerIf(scalarNarrowerThan(0, 32))
815 // FIXME: Fixing non-power-of-2 before clamp is workaround for
816 // narrowScalar limitation.
817 .widenScalarToNextPow2(0)
822 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
825 .widenScalarToNextPow2(0)
830 getActionDefinitionsBuilder(G_INTTOPTR)
831 // List the common cases
832 .legalForCartesianProduct(AddrSpaces64, {S64})
833 .legalForCartesianProduct(AddrSpaces32, {S32})
835 // Accept any address space as long as the size matches
836 .legalIf(sameSize(0, 1))
837 .widenScalarIf(smallerThan(1, 0),
838 [](const LegalityQuery &Query) {
839 return std::make_pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
841 .narrowScalarIf(largerThan(1, 0),
842 [](const LegalityQuery &Query) {
843 return std::make_pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
846 getActionDefinitionsBuilder(G_PTRTOINT)
847 // List the common cases
848 .legalForCartesianProduct(AddrSpaces64, {S64})
849 .legalForCartesianProduct(AddrSpaces32, {S32})
851 // Accept any address space as long as the size matches
852 .legalIf(sameSize(0, 1))
853 .widenScalarIf(smallerThan(0, 1),
854 [](const LegalityQuery &Query) {
855 return std::make_pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
859 [](const LegalityQuery &Query) {
860 return std::make_pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
863 getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
867 const auto needToSplitMemOp = [=](const LegalityQuery &Query,
868 bool IsLoad) -> bool {
869 const LLT DstTy = Query.Types[0];
871 // Split vector extloads.
872 unsigned MemSize = Query.MMODescrs[0].SizeInBits;
873 unsigned Align = Query.MMODescrs[0].AlignInBits;
875 if (MemSize < DstTy.getSizeInBits())
876 MemSize = std::max(MemSize, Align);
878 if (DstTy.isVector() && DstTy.getSizeInBits() > MemSize)
881 const LLT PtrTy = Query.Types[1];
882 unsigned AS = PtrTy.getAddressSpace();
883 if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad))
886 // Catch weird sized loads that don't evenly divide into the access sizes
887 // TODO: May be able to widen depending on alignment etc.
888 unsigned NumRegs = (MemSize + 31) / 32;
890 if (!ST.hasDwordx3LoadStores())
893 // If the alignment allows, these should have been widened.
894 if (!isPowerOf2_32(NumRegs))
898 if (Align < MemSize) {
899 const SITargetLowering *TLI = ST.getTargetLowering();
900 return !TLI->allowsMisalignedMemoryAccessesImpl(MemSize, AS, Align / 8);
906 const auto shouldWidenLoadResult = [=](const LegalityQuery &Query,
907 unsigned Opc) -> bool {
908 unsigned Size = Query.Types[0].getSizeInBits();
909 if (isPowerOf2_32(Size))
912 if (Size == 96 && ST.hasDwordx3LoadStores())
915 unsigned AddrSpace = Query.Types[1].getAddressSpace();
916 if (Size >= maxSizeForAddrSpace(ST, AddrSpace, Opc))
919 unsigned Align = Query.MMODescrs[0].AlignInBits;
920 unsigned RoundedSize = NextPowerOf2(Size);
921 return (Align >= RoundedSize);
924 unsigned GlobalAlign32 = ST.hasUnalignedBufferAccess() ? 0 : 32;
925 unsigned GlobalAlign16 = ST.hasUnalignedBufferAccess() ? 0 : 16;
926 unsigned GlobalAlign8 = ST.hasUnalignedBufferAccess() ? 0 : 8;
928 // TODO: Refine based on subtargets which support unaligned access or 128-bit
930 // TODO: Unsupported flat for SI.
932 for (unsigned Op : {G_LOAD, G_STORE}) {
933 const bool IsStore = Op == G_STORE;
935 auto &Actions = getActionDefinitionsBuilder(Op);
936 // Explicitly list some common cases.
937 // TODO: Does this help compile time at all?
938 Actions.legalForTypesWithMemDesc({{S32, GlobalPtr, 32, GlobalAlign32},
939 {V2S32, GlobalPtr, 64, GlobalAlign32},
940 {V4S32, GlobalPtr, 128, GlobalAlign32},
941 {S64, GlobalPtr, 64, GlobalAlign32},
942 {V2S64, GlobalPtr, 128, GlobalAlign32},
943 {V2S16, GlobalPtr, 32, GlobalAlign32},
944 {S32, GlobalPtr, 8, GlobalAlign8},
945 {S32, GlobalPtr, 16, GlobalAlign16},
947 {S32, LocalPtr, 32, 32},
948 {S64, LocalPtr, 64, 32},
949 {V2S32, LocalPtr, 64, 32},
950 {S32, LocalPtr, 8, 8},
951 {S32, LocalPtr, 16, 16},
952 {V2S16, LocalPtr, 32, 32},
954 {S32, PrivatePtr, 32, 32},
955 {S32, PrivatePtr, 8, 8},
956 {S32, PrivatePtr, 16, 16},
957 {V2S16, PrivatePtr, 32, 32},
959 {S32, ConstantPtr, 32, GlobalAlign32},
960 {V2S32, ConstantPtr, 64, GlobalAlign32},
961 {V4S32, ConstantPtr, 128, GlobalAlign32},
962 {S64, ConstantPtr, 64, GlobalAlign32},
963 {V2S32, ConstantPtr, 32, GlobalAlign32}});
965 [=](const LegalityQuery &Query) -> bool {
966 return isLoadStoreLegal(ST, Query, Op);
969 // Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
972 // TODO: Should generalize bitcast action into coerce, which will also cover
973 // inserting addrspacecasts.
974 Actions.customIf(typeIs(1, Constant32Ptr));
976 // Turn any illegal element vectors into something easier to deal
977 // with. These will ultimately produce 32-bit scalar shifts to extract the
980 // For odd 16-bit element vectors, prefer to split those into pieces with
981 // 16-bit vector parts.
983 [=](const LegalityQuery &Query) -> bool {
984 const LLT Ty = Query.Types[0];
985 const unsigned Size = Ty.getSizeInBits();
987 if (Size != Query.MMODescrs[0].SizeInBits)
988 return Size <= 32 && Ty.isVector();
990 if (loadStoreBitcastWorkaround(Ty) && isRegisterType(Ty))
992 return Ty.isVector() && (Size <= 32 || isRegisterSize(Size)) &&
993 !isRegisterVectorElementType(Ty.getElementType());
994 }, bitcastToRegisterType(0));
997 .customIf(typeIs(1, Constant32Ptr))
998 // Widen suitably aligned loads by loading extra elements.
999 .moreElementsIf([=](const LegalityQuery &Query) {
1000 const LLT Ty = Query.Types[0];
1001 return Op == G_LOAD && Ty.isVector() &&
1002 shouldWidenLoadResult(Query, Op);
1003 }, moreElementsToNextPow2(0))
1004 .widenScalarIf([=](const LegalityQuery &Query) {
1005 const LLT Ty = Query.Types[0];
1006 return Op == G_LOAD && !Ty.isVector() &&
1007 shouldWidenLoadResult(Query, Op);
1008 }, widenScalarOrEltToNextPow2(0))
1010 [=](const LegalityQuery &Query) -> bool {
1011 return !Query.Types[0].isVector() &&
1012 needToSplitMemOp(Query, Op == G_LOAD);
1014 [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
1015 const LLT DstTy = Query.Types[0];
1016 const LLT PtrTy = Query.Types[1];
1018 const unsigned DstSize = DstTy.getSizeInBits();
1019 unsigned MemSize = Query.MMODescrs[0].SizeInBits;
1022 if (DstSize > MemSize)
1023 return std::make_pair(0, LLT::scalar(MemSize));
1025 if (!isPowerOf2_32(DstSize)) {
1026 // We're probably decomposing an odd sized store. Try to split
1027 // to the widest type. TODO: Account for alignment. As-is it
1028 // should be OK, since the new parts will be further legalized.
1029 unsigned FloorSize = PowerOf2Floor(DstSize);
1030 return std::make_pair(0, LLT::scalar(FloorSize));
1033 if (DstSize > 32 && (DstSize % 32 != 0)) {
1034 // FIXME: Need a way to specify non-extload of larger size if
1035 // suitably aligned.
1036 return std::make_pair(0, LLT::scalar(32 * (DstSize / 32)));
1039 unsigned MaxSize = maxSizeForAddrSpace(ST,
1040 PtrTy.getAddressSpace(),
1042 if (MemSize > MaxSize)
1043 return std::make_pair(0, LLT::scalar(MaxSize));
1045 unsigned Align = Query.MMODescrs[0].AlignInBits;
1046 return std::make_pair(0, LLT::scalar(Align));
1049 [=](const LegalityQuery &Query) -> bool {
1050 return Query.Types[0].isVector() &&
1051 needToSplitMemOp(Query, Op == G_LOAD);
1053 [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
1054 const LLT DstTy = Query.Types[0];
1055 const LLT PtrTy = Query.Types[1];
1057 LLT EltTy = DstTy.getElementType();
1058 unsigned MaxSize = maxSizeForAddrSpace(ST,
1059 PtrTy.getAddressSpace(),
1062 // FIXME: Handle widened to power of 2 results better. This ends
1064 // FIXME: 3 element stores scalarized on SI
1066 // Split if it's too large for the address space.
1067 if (Query.MMODescrs[0].SizeInBits > MaxSize) {
1068 unsigned NumElts = DstTy.getNumElements();
1069 unsigned EltSize = EltTy.getSizeInBits();
1071 if (MaxSize % EltSize == 0) {
1072 return std::make_pair(
1073 0, LLT::scalarOrVector(MaxSize / EltSize, EltTy));
1076 unsigned NumPieces = Query.MMODescrs[0].SizeInBits / MaxSize;
1078 // FIXME: Refine when odd breakdowns handled
1079 // The scalars will need to be re-legalized.
1080 if (NumPieces == 1 || NumPieces >= NumElts ||
1081 NumElts % NumPieces != 0)
1082 return std::make_pair(0, EltTy);
1084 return std::make_pair(0,
1085 LLT::vector(NumElts / NumPieces, EltTy));
1088 // FIXME: We could probably handle weird extending loads better.
1089 unsigned MemSize = Query.MMODescrs[0].SizeInBits;
1090 if (DstTy.getSizeInBits() > MemSize)
1091 return std::make_pair(0, EltTy);
1093 unsigned EltSize = EltTy.getSizeInBits();
1094 unsigned DstSize = DstTy.getSizeInBits();
1095 if (!isPowerOf2_32(DstSize)) {
1096 // We're probably decomposing an odd sized store. Try to split
1097 // to the widest type. TODO: Account for alignment. As-is it
1098 // should be OK, since the new parts will be further legalized.
1099 unsigned FloorSize = PowerOf2Floor(DstSize);
1100 return std::make_pair(
1101 0, LLT::scalarOrVector(FloorSize / EltSize, EltTy));
1104 // Need to split because of alignment.
1105 unsigned Align = Query.MMODescrs[0].AlignInBits;
1106 if (EltSize > Align &&
1107 (EltSize / Align < DstTy.getNumElements())) {
1108 return std::make_pair(0, LLT::vector(EltSize / Align, EltTy));
1111 // May need relegalization for the scalars.
1112 return std::make_pair(0, EltTy);
1117 Actions.narrowScalarIf(isWideScalarTruncStore(0), changeTo(0, S32));
1119 // TODO: Need a bitcast lower option?
1121 .widenScalarToNextPow2(0)
1122 .moreElementsIf(vectorSmallerThan(0, 32), moreEltsToNext32Bit(0));
1125 auto &ExtLoads = getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
1126 .legalForTypesWithMemDesc({{S32, GlobalPtr, 8, 8},
1127 {S32, GlobalPtr, 16, 2 * 8},
1128 {S32, LocalPtr, 8, 8},
1129 {S32, LocalPtr, 16, 16},
1130 {S32, PrivatePtr, 8, 8},
1131 {S32, PrivatePtr, 16, 16},
1132 {S32, ConstantPtr, 8, 8},
1133 {S32, ConstantPtr, 16, 2 * 8}});
1134 if (ST.hasFlatAddressSpace()) {
1135 ExtLoads.legalForTypesWithMemDesc(
1136 {{S32, FlatPtr, 8, 8}, {S32, FlatPtr, 16, 16}});
1139 ExtLoads.clampScalar(0, S32, S32)
1140 .widenScalarToNextPow2(0)
1141 .unsupportedIfMemSizeNotPow2()
1144 auto &Atomics = getActionDefinitionsBuilder(
1145 {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB,
1146 G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
1147 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
1149 .legalFor({{S32, GlobalPtr}, {S32, LocalPtr},
1150 {S64, GlobalPtr}, {S64, LocalPtr}});
1151 if (ST.hasFlatAddressSpace()) {
1152 Atomics.legalFor({{S32, FlatPtr}, {S64, FlatPtr}});
1155 if (ST.hasLDSFPAtomics()) {
1156 getActionDefinitionsBuilder(G_ATOMICRMW_FADD)
1157 .legalFor({{S32, LocalPtr}});
1160 // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling, and output
1162 getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG)
1163 .customFor({{S32, GlobalPtr}, {S64, GlobalPtr},
1164 {S32, FlatPtr}, {S64, FlatPtr}})
1165 .legalFor({{S32, LocalPtr}, {S64, LocalPtr},
1166 {S32, RegionPtr}, {S64, RegionPtr}});
1167 // TODO: Pointer types, any 32-bit or 64-bit vector
1169 // Condition should be s32 for scalar, s1 for vector.
1170 getActionDefinitionsBuilder(G_SELECT)
1171 .legalForCartesianProduct({S32, S64, S16, V2S32, V2S16, V4S16,
1172 GlobalPtr, LocalPtr, FlatPtr, PrivatePtr,
1173 LLT::vector(2, LocalPtr), LLT::vector(2, PrivatePtr)}, {S1, S32})
1174 .clampScalar(0, S16, S64)
1176 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
1177 .fewerElementsIf(numElementsNotEven(0), scalarize(0))
1178 .clampMaxNumElements(0, S32, 2)
1179 .clampMaxNumElements(0, LocalPtr, 2)
1180 .clampMaxNumElements(0, PrivatePtr, 2)
1182 .widenScalarToNextPow2(0)
1183 .legalIf(all(isPointer(0), typeInSet(1, {S1, S32})));
1185 // TODO: Only the low 4/5/6 bits of the shift amount are observed, so we can
1186 // be more flexible with the shift amount type.
1187 auto &Shifts = getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})
1188 .legalFor({{S32, S32}, {S64, S32}});
1189 if (ST.has16BitInsts()) {
1190 if (ST.hasVOP3PInsts()) {
1191 Shifts.legalFor({{S16, S16}, {V2S16, V2S16}})
1192 .clampMaxNumElements(0, S16, 2);
1194 Shifts.legalFor({{S16, S16}});
1196 // TODO: Support 16-bit shift amounts for all types
1197 Shifts.widenScalarIf(
1198 [=](const LegalityQuery &Query) {
1199 // Use 16-bit shift amounts for any 16-bit shift. Otherwise we want a
1201 const LLT ValTy = Query.Types[0];
1202 const LLT AmountTy = Query.Types[1];
1203 return ValTy.getSizeInBits() <= 16 &&
1204 AmountTy.getSizeInBits() < 16;
1205 }, changeTo(1, S16));
1206 Shifts.maxScalarIf(typeIs(0, S16), 1, S16);
1207 Shifts.clampScalar(1, S32, S32);
1208 Shifts.clampScalar(0, S16, S64);
1209 Shifts.widenScalarToNextPow2(0, 16);
1211 // Make sure we legalize the shift amount type first, as the general
1212 // expansion for the shifted type will produce much worse code if it hasn't
1213 // been truncated already.
1214 Shifts.clampScalar(1, S32, S32);
1215 Shifts.clampScalar(0, S32, S64);
1216 Shifts.widenScalarToNextPow2(0, 32);
1218 Shifts.scalarize(0);
1220 for (unsigned Op : {G_EXTRACT_VECTOR_ELT, G_INSERT_VECTOR_ELT}) {
1221 unsigned VecTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 1 : 0;
1222 unsigned EltTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 0 : 1;
1223 unsigned IdxTypeIdx = 2;
1225 getActionDefinitionsBuilder(Op)
1226 .customIf([=](const LegalityQuery &Query) {
1227 const LLT EltTy = Query.Types[EltTypeIdx];
1228 const LLT VecTy = Query.Types[VecTypeIdx];
1229 const LLT IdxTy = Query.Types[IdxTypeIdx];
1230 return (EltTy.getSizeInBits() == 16 ||
1231 EltTy.getSizeInBits() % 32 == 0) &&
1232 VecTy.getSizeInBits() % 32 == 0 &&
1233 VecTy.getSizeInBits() <= MaxRegisterSize &&
1234 IdxTy.getSizeInBits() == 32;
1236 .clampScalar(EltTypeIdx, S32, S64)
1237 .clampScalar(VecTypeIdx, S32, S64)
1238 .clampScalar(IdxTypeIdx, S32, S32);
1241 getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
1242 .unsupportedIf([=](const LegalityQuery &Query) {
1243 const LLT &EltTy = Query.Types[1].getElementType();
1244 return Query.Types[0] != EltTy;
1247 for (unsigned Op : {G_EXTRACT, G_INSERT}) {
1248 unsigned BigTyIdx = Op == G_EXTRACT ? 1 : 0;
1249 unsigned LitTyIdx = Op == G_EXTRACT ? 0 : 1;
1251 // FIXME: Doesn't handle extract of illegal sizes.
1252 getActionDefinitionsBuilder(Op)
1253 .lowerIf(all(typeIs(LitTyIdx, S16), sizeIs(BigTyIdx, 32)))
1254 // FIXME: Multiples of 16 should not be legal.
1255 .legalIf([=](const LegalityQuery &Query) {
1256 const LLT BigTy = Query.Types[BigTyIdx];
1257 const LLT LitTy = Query.Types[LitTyIdx];
1258 return (BigTy.getSizeInBits() % 32 == 0) &&
1259 (LitTy.getSizeInBits() % 16 == 0);
1262 [=](const LegalityQuery &Query) {
1263 const LLT BigTy = Query.Types[BigTyIdx];
1264 return (BigTy.getScalarSizeInBits() < 16);
1266 LegalizeMutations::widenScalarOrEltToNextPow2(BigTyIdx, 16))
1268 [=](const LegalityQuery &Query) {
1269 const LLT LitTy = Query.Types[LitTyIdx];
1270 return (LitTy.getScalarSizeInBits() < 16);
1272 LegalizeMutations::widenScalarOrEltToNextPow2(LitTyIdx, 16))
1273 .moreElementsIf(isSmallOddVector(BigTyIdx), oneMoreElement(BigTyIdx))
1274 .widenScalarToNextPow2(BigTyIdx, 32);
1278 auto &BuildVector = getActionDefinitionsBuilder(G_BUILD_VECTOR)
1279 .legalForCartesianProduct(AllS32Vectors, {S32})
1280 .legalForCartesianProduct(AllS64Vectors, {S64})
1281 .clampNumElements(0, V16S32, V32S32)
1282 .clampNumElements(0, V2S64, V16S64)
1283 .fewerElementsIf(isWideVec16(0), changeTo(0, V2S16));
1285 if (ST.hasScalarPackInsts()) {
1287 // FIXME: Should probably widen s1 vectors straight to s32
1288 .minScalarOrElt(0, S16)
1289 // Widen source elements and produce a G_BUILD_VECTOR_TRUNC
1292 getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
1293 .legalFor({V2S16, S32})
1295 BuildVector.minScalarOrElt(0, S32);
1297 BuildVector.customFor({V2S16, S16});
1298 BuildVector.minScalarOrElt(0, S32);
1300 getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
1301 .customFor({V2S16, S32})
1305 BuildVector.legalIf(isRegisterType(0));
1307 // FIXME: Clamp maximum size
1308 getActionDefinitionsBuilder(G_CONCAT_VECTORS)
1309 .legalIf(isRegisterType(0));
1311 // TODO: Don't fully scalarize v2s16 pieces? Or combine out thosse
1313 if (ST.hasVOP3PInsts()) {
1314 getActionDefinitionsBuilder(G_SHUFFLE_VECTOR)
1315 .customFor({V2S16, V2S16})
1318 getActionDefinitionsBuilder(G_SHUFFLE_VECTOR).lower();
1321 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
1322 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
1323 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
1325 auto notValidElt = [=](const LegalityQuery &Query, unsigned TypeIdx) {
1326 const LLT Ty = Query.Types[TypeIdx];
1327 if (Ty.isVector()) {
1328 const LLT &EltTy = Ty.getElementType();
1329 if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 512)
1331 if (!isPowerOf2_32(EltTy.getSizeInBits()))
1337 auto &Builder = getActionDefinitionsBuilder(Op)
1338 .lowerFor({{S16, V2S16}})
1339 .lowerIf([=](const LegalityQuery &Query) {
1340 const LLT BigTy = Query.Types[BigTyIdx];
1341 return BigTy.getSizeInBits() == 32;
1343 // Try to widen to s16 first for small types.
1344 // TODO: Only do this on targets with legal s16 shifts
1345 .minScalarOrEltIf(scalarNarrowerThan(LitTyIdx, 16), LitTyIdx, S16)
1346 .widenScalarToNextPow2(LitTyIdx, /*Min*/ 16)
1347 .moreElementsIf(isSmallOddVector(BigTyIdx), oneMoreElement(BigTyIdx))
1348 .fewerElementsIf(all(typeIs(0, S16), vectorWiderThan(1, 32),
1349 elementTypeIs(1, S16)),
1351 // Clamp the little scalar to s8-s256 and make it a power of 2. It's not
1352 // worth considering the multiples of 64 since 2*192 and 2*384 are not
1354 .clampScalar(LitTyIdx, S32, S512)
1355 .widenScalarToNextPow2(LitTyIdx, /*Min*/ 32)
1356 // Break up vectors with weird elements into scalars
1358 [=](const LegalityQuery &Query) { return notValidElt(Query, LitTyIdx); },
1361 [=](const LegalityQuery &Query) { return notValidElt(Query, BigTyIdx); },
1363 .clampScalar(BigTyIdx, S32, MaxScalar);
1365 if (Op == G_MERGE_VALUES) {
1366 Builder.widenScalarIf(
1367 // TODO: Use 16-bit shifts if legal for 8-bit values?
1368 [=](const LegalityQuery &Query) {
1369 const LLT Ty = Query.Types[LitTyIdx];
1370 return Ty.getSizeInBits() < 32;
1372 changeTo(LitTyIdx, S32));
1375 Builder.widenScalarIf(
1376 [=](const LegalityQuery &Query) {
1377 const LLT Ty = Query.Types[BigTyIdx];
1378 return !isPowerOf2_32(Ty.getSizeInBits()) &&
1379 Ty.getSizeInBits() % 16 != 0;
1381 [=](const LegalityQuery &Query) {
1382 // Pick the next power of 2, or a multiple of 64 over 128.
1383 // Whichever is smaller.
1384 const LLT &Ty = Query.Types[BigTyIdx];
1385 unsigned NewSizeInBits = 1 << Log2_32_Ceil(Ty.getSizeInBits() + 1);
1386 if (NewSizeInBits >= 256) {
1387 unsigned RoundedTo = alignTo<64>(Ty.getSizeInBits() + 1);
1388 if (RoundedTo < NewSizeInBits)
1389 NewSizeInBits = RoundedTo;
1391 return std::make_pair(BigTyIdx, LLT::scalar(NewSizeInBits));
1393 .legalIf([=](const LegalityQuery &Query) {
1394 const LLT &BigTy = Query.Types[BigTyIdx];
1395 const LLT &LitTy = Query.Types[LitTyIdx];
1397 if (BigTy.isVector() && BigTy.getSizeInBits() < 32)
1399 if (LitTy.isVector() && LitTy.getSizeInBits() < 32)
1402 return BigTy.getSizeInBits() % 16 == 0 &&
1403 LitTy.getSizeInBits() % 16 == 0 &&
1404 BigTy.getSizeInBits() <= MaxRegisterSize;
1406 // Any vectors left are the wrong size. Scalarize them.
1411 // S64 is only legal on SALU, and needs to be broken into 32-bit elements in
1413 auto &SextInReg = getActionDefinitionsBuilder(G_SEXT_INREG)
1414 .legalFor({{S32}, {S64}});
1416 if (ST.hasVOP3PInsts()) {
1417 SextInReg.lowerFor({{V2S16}})
1418 // Prefer to reduce vector widths for 16-bit vectors before lowering, to
1419 // get more vector shift opportunities, since we'll get those when
1421 .fewerElementsIf(elementTypeIs(0, S16), changeTo(0, V2S16));
1422 } else if (ST.has16BitInsts()) {
1423 SextInReg.lowerFor({{S32}, {S64}, {S16}});
1425 // Prefer to promote to s32 before lowering if we don't have 16-bit
1426 // shifts. This avoid a lot of intermediate truncate and extend operations.
1427 SextInReg.lowerFor({{S32}, {S64}});
1430 // FIXME: Placeholder rule. Really depends on whether the clamp modifier is
1431 // available, and is selectively legal for s16, s32, v2s16.
1432 getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT, G_UADDSAT, G_USUBSAT})
1434 .clampScalar(0, S16, S32);
1438 .clampScalar(0, S32, S64)
1441 getActionDefinitionsBuilder(G_FSHR)
1442 .legalFor({{S32, S32}})
1446 getActionDefinitionsBuilder(G_READCYCLECOUNTER)
1449 getActionDefinitionsBuilder({
1450 // TODO: Verify V_BFI_B32 is generated from expanded bit ops
1453 G_ATOMIC_CMPXCHG_WITH_SUCCESS,
1460 G_FMINIMUM, G_FMAXIMUM,
1464 getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE,
1465 G_INDEXED_LOAD, G_INDEXED_SEXTLOAD,
1466 G_INDEXED_ZEXTLOAD, G_INDEXED_STORE})
1470 verify(*ST.getInstrInfo());
1473 bool AMDGPULegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
1474 MachineInstr &MI) const {
1475 MachineIRBuilder &B = Helper.MIRBuilder;
1476 MachineRegisterInfo &MRI = *B.getMRI();
1477 GISelChangeObserver &Observer = Helper.Observer;
1479 switch (MI.getOpcode()) {
1480 case TargetOpcode::G_ADDRSPACE_CAST:
1481 return legalizeAddrSpaceCast(MI, MRI, B);
1482 case TargetOpcode::G_FRINT:
1483 return legalizeFrint(MI, MRI, B);
1484 case TargetOpcode::G_FCEIL:
1485 return legalizeFceil(MI, MRI, B);
1486 case TargetOpcode::G_INTRINSIC_TRUNC:
1487 return legalizeIntrinsicTrunc(MI, MRI, B);
1488 case TargetOpcode::G_SITOFP:
1489 return legalizeITOFP(MI, MRI, B, true);
1490 case TargetOpcode::G_UITOFP:
1491 return legalizeITOFP(MI, MRI, B, false);
1492 case TargetOpcode::G_FPTOSI:
1493 return legalizeFPTOI(MI, MRI, B, true);
1494 case TargetOpcode::G_FPTOUI:
1495 return legalizeFPTOI(MI, MRI, B, false);
1496 case TargetOpcode::G_FMINNUM:
1497 case TargetOpcode::G_FMAXNUM:
1498 case TargetOpcode::G_FMINNUM_IEEE:
1499 case TargetOpcode::G_FMAXNUM_IEEE:
1500 return legalizeMinNumMaxNum(Helper, MI);
1501 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1502 return legalizeExtractVectorElt(MI, MRI, B);
1503 case TargetOpcode::G_INSERT_VECTOR_ELT:
1504 return legalizeInsertVectorElt(MI, MRI, B);
1505 case TargetOpcode::G_SHUFFLE_VECTOR:
1506 return legalizeShuffleVector(MI, MRI, B);
1507 case TargetOpcode::G_FSIN:
1508 case TargetOpcode::G_FCOS:
1509 return legalizeSinCos(MI, MRI, B);
1510 case TargetOpcode::G_GLOBAL_VALUE:
1511 return legalizeGlobalValue(MI, MRI, B);
1512 case TargetOpcode::G_LOAD:
1513 return legalizeLoad(MI, MRI, B, Observer);
1514 case TargetOpcode::G_FMAD:
1515 return legalizeFMad(MI, MRI, B);
1516 case TargetOpcode::G_FDIV:
1517 return legalizeFDIV(MI, MRI, B);
1518 case TargetOpcode::G_UDIV:
1519 case TargetOpcode::G_UREM:
1520 return legalizeUDIV_UREM(MI, MRI, B);
1521 case TargetOpcode::G_SDIV:
1522 case TargetOpcode::G_SREM:
1523 return legalizeSDIV_SREM(MI, MRI, B);
1524 case TargetOpcode::G_ATOMIC_CMPXCHG:
1525 return legalizeAtomicCmpXChg(MI, MRI, B);
1526 case TargetOpcode::G_FLOG:
1527 return legalizeFlog(MI, B, numbers::ln2f);
1528 case TargetOpcode::G_FLOG10:
1529 return legalizeFlog(MI, B, numbers::ln2f / numbers::ln10f);
1530 case TargetOpcode::G_FEXP:
1531 return legalizeFExp(MI, B);
1532 case TargetOpcode::G_FPOW:
1533 return legalizeFPow(MI, B);
1534 case TargetOpcode::G_FFLOOR:
1535 return legalizeFFloor(MI, MRI, B);
1536 case TargetOpcode::G_BUILD_VECTOR:
1537 return legalizeBuildVector(MI, MRI, B);
1542 llvm_unreachable("expected switch to return");
1545 Register AMDGPULegalizerInfo::getSegmentAperture(
1547 MachineRegisterInfo &MRI,
1548 MachineIRBuilder &B) const {
1549 MachineFunction &MF = B.getMF();
1550 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1551 const LLT S32 = LLT::scalar(32);
1553 assert(AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS);
1555 if (ST.hasApertureRegs()) {
1556 // FIXME: Use inline constants (src_{shared, private}_base) instead of
1558 unsigned Offset = AS == AMDGPUAS::LOCAL_ADDRESS ?
1559 AMDGPU::Hwreg::OFFSET_SRC_SHARED_BASE :
1560 AMDGPU::Hwreg::OFFSET_SRC_PRIVATE_BASE;
1561 unsigned WidthM1 = AS == AMDGPUAS::LOCAL_ADDRESS ?
1562 AMDGPU::Hwreg::WIDTH_M1_SRC_SHARED_BASE :
1563 AMDGPU::Hwreg::WIDTH_M1_SRC_PRIVATE_BASE;
1565 AMDGPU::Hwreg::ID_MEM_BASES << AMDGPU::Hwreg::ID_SHIFT_ |
1566 Offset << AMDGPU::Hwreg::OFFSET_SHIFT_ |
1567 WidthM1 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_;
1569 Register GetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
1571 B.buildInstr(AMDGPU::S_GETREG_B32)
1574 MRI.setType(GetReg, S32);
1576 auto ShiftAmt = B.buildConstant(S32, WidthM1 + 1);
1577 return B.buildShl(S32, GetReg, ShiftAmt).getReg(0);
1580 Register QueuePtr = MRI.createGenericVirtualRegister(
1581 LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
1583 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1584 if (!loadInputValue(QueuePtr, B, &MFI->getArgInfo().QueuePtr))
1587 // Offset into amd_queue_t for group_segment_aperture_base_hi /
1588 // private_segment_aperture_base_hi.
1589 uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44;
1591 // TODO: can we be smarter about machine pointer info?
1592 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
1593 MachineMemOperand *MMO = MF.getMachineMemOperand(
1595 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1596 MachineMemOperand::MOInvariant,
1597 4, commonAlignment(Align(64), StructOffset));
1601 B.materializePtrAdd(LoadAddr, QueuePtr, LLT::scalar(64), StructOffset);
1602 return B.buildLoad(S32, LoadAddr, *MMO).getReg(0);
1605 bool AMDGPULegalizerInfo::legalizeAddrSpaceCast(
1606 MachineInstr &MI, MachineRegisterInfo &MRI,
1607 MachineIRBuilder &B) const {
1608 MachineFunction &MF = B.getMF();
1610 const LLT S32 = LLT::scalar(32);
1611 Register Dst = MI.getOperand(0).getReg();
1612 Register Src = MI.getOperand(1).getReg();
1614 LLT DstTy = MRI.getType(Dst);
1615 LLT SrcTy = MRI.getType(Src);
1616 unsigned DestAS = DstTy.getAddressSpace();
1617 unsigned SrcAS = SrcTy.getAddressSpace();
1619 // TODO: Avoid reloading from the queue ptr for each cast, or at least each
1621 assert(!DstTy.isVector());
1623 const AMDGPUTargetMachine &TM
1624 = static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
1626 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1627 if (ST.getTargetLowering()->isNoopAddrSpaceCast(SrcAS, DestAS)) {
1628 MI.setDesc(B.getTII().get(TargetOpcode::G_BITCAST));
1632 if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
1634 B.buildExtract(Dst, Src, 0);
1635 MI.eraseFromParent();
1639 if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
1640 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1641 uint32_t AddrHiVal = Info->get32BitAddressHighBits();
1643 // FIXME: This is a bit ugly due to creating a merge of 2 pointers to
1644 // another. Merge operands are required to be the same type, but creating an
1645 // extra ptrtoint would be kind of pointless.
1646 auto HighAddr = B.buildConstant(
1647 LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS_32BIT, 32), AddrHiVal);
1648 B.buildMerge(Dst, {Src, HighAddr});
1649 MI.eraseFromParent();
1653 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
1654 assert(DestAS == AMDGPUAS::LOCAL_ADDRESS ||
1655 DestAS == AMDGPUAS::PRIVATE_ADDRESS);
1656 unsigned NullVal = TM.getNullPointerValue(DestAS);
1658 auto SegmentNull = B.buildConstant(DstTy, NullVal);
1659 auto FlatNull = B.buildConstant(SrcTy, 0);
1661 // Extract low 32-bits of the pointer.
1662 auto PtrLo32 = B.buildExtract(DstTy, Src, 0);
1665 B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src, FlatNull.getReg(0));
1666 B.buildSelect(Dst, CmpRes, PtrLo32, SegmentNull.getReg(0));
1668 MI.eraseFromParent();
1672 if (SrcAS != AMDGPUAS::LOCAL_ADDRESS && SrcAS != AMDGPUAS::PRIVATE_ADDRESS)
1675 if (!ST.hasFlatAddressSpace())
1679 B.buildConstant(SrcTy, TM.getNullPointerValue(SrcAS));
1681 B.buildConstant(DstTy, TM.getNullPointerValue(DestAS));
1683 Register ApertureReg = getSegmentAperture(SrcAS, MRI, B);
1684 if (!ApertureReg.isValid())
1688 B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src, SegmentNull.getReg(0));
1690 // Coerce the type of the low half of the result so we can use merge_values.
1691 Register SrcAsInt = B.buildPtrToInt(S32, Src).getReg(0);
1693 // TODO: Should we allow mismatched types but matching sizes in merges to
1694 // avoid the ptrtoint?
1695 auto BuildPtr = B.buildMerge(DstTy, {SrcAsInt, ApertureReg});
1696 B.buildSelect(Dst, CmpRes, BuildPtr, FlatNull);
1698 MI.eraseFromParent();
1702 bool AMDGPULegalizerInfo::legalizeFrint(
1703 MachineInstr &MI, MachineRegisterInfo &MRI,
1704 MachineIRBuilder &B) const {
1705 Register Src = MI.getOperand(1).getReg();
1706 LLT Ty = MRI.getType(Src);
1707 assert(Ty.isScalar() && Ty.getSizeInBits() == 64);
1709 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
1710 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
1712 auto C1 = B.buildFConstant(Ty, C1Val);
1713 auto CopySign = B.buildFCopysign(Ty, C1, Src);
1715 // TODO: Should this propagate fast-math-flags?
1716 auto Tmp1 = B.buildFAdd(Ty, Src, CopySign);
1717 auto Tmp2 = B.buildFSub(Ty, Tmp1, CopySign);
1719 auto C2 = B.buildFConstant(Ty, C2Val);
1720 auto Fabs = B.buildFAbs(Ty, Src);
1722 auto Cond = B.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), Fabs, C2);
1723 B.buildSelect(MI.getOperand(0).getReg(), Cond, Src, Tmp2);
1727 bool AMDGPULegalizerInfo::legalizeFceil(
1728 MachineInstr &MI, MachineRegisterInfo &MRI,
1729 MachineIRBuilder &B) const {
1731 const LLT S1 = LLT::scalar(1);
1732 const LLT S64 = LLT::scalar(64);
1734 Register Src = MI.getOperand(1).getReg();
1735 assert(MRI.getType(Src) == S64);
1737 // result = trunc(src)
1738 // if (src > 0.0 && src != result)
1741 auto Trunc = B.buildIntrinsicTrunc(S64, Src);
1743 const auto Zero = B.buildFConstant(S64, 0.0);
1744 const auto One = B.buildFConstant(S64, 1.0);
1745 auto Lt0 = B.buildFCmp(CmpInst::FCMP_OGT, S1, Src, Zero);
1746 auto NeTrunc = B.buildFCmp(CmpInst::FCMP_ONE, S1, Src, Trunc);
1747 auto And = B.buildAnd(S1, Lt0, NeTrunc);
1748 auto Add = B.buildSelect(S64, And, One, Zero);
1750 // TODO: Should this propagate fast-math-flags?
1751 B.buildFAdd(MI.getOperand(0).getReg(), Trunc, Add);
1755 static MachineInstrBuilder extractF64Exponent(unsigned Hi,
1756 MachineIRBuilder &B) {
1757 const unsigned FractBits = 52;
1758 const unsigned ExpBits = 11;
1759 LLT S32 = LLT::scalar(32);
1761 auto Const0 = B.buildConstant(S32, FractBits - 32);
1762 auto Const1 = B.buildConstant(S32, ExpBits);
1764 auto ExpPart = B.buildIntrinsic(Intrinsic::amdgcn_ubfe, {S32}, false)
1765 .addUse(Const0.getReg(0))
1766 .addUse(Const1.getReg(0));
1768 return B.buildSub(S32, ExpPart, B.buildConstant(S32, 1023));
1771 bool AMDGPULegalizerInfo::legalizeIntrinsicTrunc(
1772 MachineInstr &MI, MachineRegisterInfo &MRI,
1773 MachineIRBuilder &B) const {
1774 const LLT S1 = LLT::scalar(1);
1775 const LLT S32 = LLT::scalar(32);
1776 const LLT S64 = LLT::scalar(64);
1778 Register Src = MI.getOperand(1).getReg();
1779 assert(MRI.getType(Src) == S64);
1781 // TODO: Should this use extract since the low half is unused?
1782 auto Unmerge = B.buildUnmerge({S32, S32}, Src);
1783 Register Hi = Unmerge.getReg(1);
1785 // Extract the upper half, since this is where we will find the sign and
1787 auto Exp = extractF64Exponent(Hi, B);
1789 const unsigned FractBits = 52;
1791 // Extract the sign bit.
1792 const auto SignBitMask = B.buildConstant(S32, UINT32_C(1) << 31);
1793 auto SignBit = B.buildAnd(S32, Hi, SignBitMask);
1795 const auto FractMask = B.buildConstant(S64, (UINT64_C(1) << FractBits) - 1);
1797 const auto Zero32 = B.buildConstant(S32, 0);
1799 // Extend back to 64-bits.
1800 auto SignBit64 = B.buildMerge(S64, {Zero32, SignBit});
1802 auto Shr = B.buildAShr(S64, FractMask, Exp);
1803 auto Not = B.buildNot(S64, Shr);
1804 auto Tmp0 = B.buildAnd(S64, Src, Not);
1805 auto FiftyOne = B.buildConstant(S32, FractBits - 1);
1807 auto ExpLt0 = B.buildICmp(CmpInst::ICMP_SLT, S1, Exp, Zero32);
1808 auto ExpGt51 = B.buildICmp(CmpInst::ICMP_SGT, S1, Exp, FiftyOne);
1810 auto Tmp1 = B.buildSelect(S64, ExpLt0, SignBit64, Tmp0);
1811 B.buildSelect(MI.getOperand(0).getReg(), ExpGt51, Src, Tmp1);
1815 bool AMDGPULegalizerInfo::legalizeITOFP(
1816 MachineInstr &MI, MachineRegisterInfo &MRI,
1817 MachineIRBuilder &B, bool Signed) const {
1819 Register Dst = MI.getOperand(0).getReg();
1820 Register Src = MI.getOperand(1).getReg();
1822 const LLT S64 = LLT::scalar(64);
1823 const LLT S32 = LLT::scalar(32);
1825 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S64);
1827 auto Unmerge = B.buildUnmerge({S32, S32}, Src);
1829 auto CvtHi = Signed ?
1830 B.buildSITOFP(S64, Unmerge.getReg(1)) :
1831 B.buildUITOFP(S64, Unmerge.getReg(1));
1833 auto CvtLo = B.buildUITOFP(S64, Unmerge.getReg(0));
1835 auto ThirtyTwo = B.buildConstant(S32, 32);
1836 auto LdExp = B.buildIntrinsic(Intrinsic::amdgcn_ldexp, {S64}, false)
1837 .addUse(CvtHi.getReg(0))
1838 .addUse(ThirtyTwo.getReg(0));
1840 // TODO: Should this propagate fast-math-flags?
1841 B.buildFAdd(Dst, LdExp, CvtLo);
1842 MI.eraseFromParent();
1846 // TODO: Copied from DAG implementation. Verify logic and document how this
1848 bool AMDGPULegalizerInfo::legalizeFPTOI(
1849 MachineInstr &MI, MachineRegisterInfo &MRI,
1850 MachineIRBuilder &B, bool Signed) const {
1852 Register Dst = MI.getOperand(0).getReg();
1853 Register Src = MI.getOperand(1).getReg();
1855 const LLT S64 = LLT::scalar(64);
1856 const LLT S32 = LLT::scalar(32);
1858 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S64);
1860 unsigned Flags = MI.getFlags();
1862 auto Trunc = B.buildIntrinsicTrunc(S64, Src, Flags);
1863 auto K0 = B.buildFConstant(S64, BitsToDouble(UINT64_C(0x3df0000000000000)));
1864 auto K1 = B.buildFConstant(S64, BitsToDouble(UINT64_C(0xc1f0000000000000)));
1866 auto Mul = B.buildFMul(S64, Trunc, K0, Flags);
1867 auto FloorMul = B.buildFFloor(S64, Mul, Flags);
1868 auto Fma = B.buildFMA(S64, FloorMul, K1, Trunc, Flags);
1871 B.buildFPTOSI(S32, FloorMul) :
1872 B.buildFPTOUI(S32, FloorMul);
1873 auto Lo = B.buildFPTOUI(S32, Fma);
1875 B.buildMerge(Dst, { Lo, Hi });
1876 MI.eraseFromParent();
1881 bool AMDGPULegalizerInfo::legalizeMinNumMaxNum(LegalizerHelper &Helper,
1882 MachineInstr &MI) const {
1883 MachineFunction &MF = Helper.MIRBuilder.getMF();
1884 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1886 const bool IsIEEEOp = MI.getOpcode() == AMDGPU::G_FMINNUM_IEEE ||
1887 MI.getOpcode() == AMDGPU::G_FMAXNUM_IEEE;
1889 // With ieee_mode disabled, the instructions have the correct behavior
1890 // already for G_FMINNUM/G_FMAXNUM
1891 if (!MFI->getMode().IEEE)
1897 return Helper.lowerFMinNumMaxNum(MI) == LegalizerHelper::Legalized;
1900 bool AMDGPULegalizerInfo::legalizeExtractVectorElt(
1901 MachineInstr &MI, MachineRegisterInfo &MRI,
1902 MachineIRBuilder &B) const {
1903 // TODO: Should move some of this into LegalizerHelper.
1905 // TODO: Promote dynamic indexing of s16 to s32
1907 // FIXME: Artifact combiner probably should have replaced the truncated
1908 // constant before this, so we shouldn't need
1909 // getConstantVRegValWithLookThrough.
1910 Optional<ValueAndVReg> IdxVal = getConstantVRegValWithLookThrough(
1911 MI.getOperand(2).getReg(), MRI);
1912 if (!IdxVal) // Dynamic case will be selected to register indexing.
1915 Register Dst = MI.getOperand(0).getReg();
1916 Register Vec = MI.getOperand(1).getReg();
1918 LLT VecTy = MRI.getType(Vec);
1919 LLT EltTy = VecTy.getElementType();
1920 assert(EltTy == MRI.getType(Dst));
1922 if (IdxVal->Value < VecTy.getNumElements())
1923 B.buildExtract(Dst, Vec, IdxVal->Value * EltTy.getSizeInBits());
1927 MI.eraseFromParent();
1931 bool AMDGPULegalizerInfo::legalizeInsertVectorElt(
1932 MachineInstr &MI, MachineRegisterInfo &MRI,
1933 MachineIRBuilder &B) const {
1934 // TODO: Should move some of this into LegalizerHelper.
1936 // TODO: Promote dynamic indexing of s16 to s32
1938 // FIXME: Artifact combiner probably should have replaced the truncated
1939 // constant before this, so we shouldn't need
1940 // getConstantVRegValWithLookThrough.
1941 Optional<ValueAndVReg> IdxVal = getConstantVRegValWithLookThrough(
1942 MI.getOperand(3).getReg(), MRI);
1943 if (!IdxVal) // Dynamic case will be selected to register indexing.
1946 Register Dst = MI.getOperand(0).getReg();
1947 Register Vec = MI.getOperand(1).getReg();
1948 Register Ins = MI.getOperand(2).getReg();
1950 LLT VecTy = MRI.getType(Vec);
1951 LLT EltTy = VecTy.getElementType();
1952 assert(EltTy == MRI.getType(Ins));
1954 if (IdxVal->Value < VecTy.getNumElements())
1955 B.buildInsert(Dst, Vec, Ins, IdxVal->Value * EltTy.getSizeInBits());
1959 MI.eraseFromParent();
1963 bool AMDGPULegalizerInfo::legalizeShuffleVector(
1964 MachineInstr &MI, MachineRegisterInfo &MRI,
1965 MachineIRBuilder &B) const {
1966 const LLT V2S16 = LLT::vector(2, 16);
1968 Register Dst = MI.getOperand(0).getReg();
1969 Register Src0 = MI.getOperand(1).getReg();
1970 LLT DstTy = MRI.getType(Dst);
1971 LLT SrcTy = MRI.getType(Src0);
1973 if (SrcTy == V2S16 && DstTy == V2S16 &&
1974 AMDGPU::isLegalVOP3PShuffleMask(MI.getOperand(3).getShuffleMask()))
1977 MachineIRBuilder HelperBuilder(MI);
1978 GISelObserverWrapper DummyObserver;
1979 LegalizerHelper Helper(B.getMF(), DummyObserver, HelperBuilder);
1980 return Helper.lowerShuffleVector(MI) == LegalizerHelper::Legalized;
1983 bool AMDGPULegalizerInfo::legalizeSinCos(
1984 MachineInstr &MI, MachineRegisterInfo &MRI,
1985 MachineIRBuilder &B) const {
1987 Register DstReg = MI.getOperand(0).getReg();
1988 Register SrcReg = MI.getOperand(1).getReg();
1989 LLT Ty = MRI.getType(DstReg);
1990 unsigned Flags = MI.getFlags();
1993 auto OneOver2Pi = B.buildFConstant(Ty, 0.5 * numbers::inv_pi);
1994 if (ST.hasTrigReducedRange()) {
1995 auto MulVal = B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags);
1996 TrigVal = B.buildIntrinsic(Intrinsic::amdgcn_fract, {Ty}, false)
1997 .addUse(MulVal.getReg(0))
1998 .setMIFlags(Flags).getReg(0);
2000 TrigVal = B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags).getReg(0);
2002 Intrinsic::ID TrigIntrin = MI.getOpcode() == AMDGPU::G_FSIN ?
2003 Intrinsic::amdgcn_sin : Intrinsic::amdgcn_cos;
2004 B.buildIntrinsic(TrigIntrin, makeArrayRef<Register>(DstReg), false)
2007 MI.eraseFromParent();
2011 bool AMDGPULegalizerInfo::buildPCRelGlobalAddress(Register DstReg, LLT PtrTy,
2012 MachineIRBuilder &B,
2013 const GlobalValue *GV,
2015 unsigned GAFlags) const {
2016 assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
2017 // In order to support pc-relative addressing, SI_PC_ADD_REL_OFFSET is lowered
2018 // to the following code sequence:
2020 // For constant address space:
2021 // s_getpc_b64 s[0:1]
2022 // s_add_u32 s0, s0, $symbol
2023 // s_addc_u32 s1, s1, 0
2025 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
2026 // a fixup or relocation is emitted to replace $symbol with a literal
2027 // constant, which is a pc-relative offset from the encoding of the $symbol
2028 // operand to the global variable.
2030 // For global address space:
2031 // s_getpc_b64 s[0:1]
2032 // s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
2033 // s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
2035 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
2036 // fixups or relocations are emitted to replace $symbol@*@lo and
2037 // $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
2038 // which is a 64-bit pc-relative offset from the encoding of the $symbol
2039 // operand to the global variable.
2041 // What we want here is an offset from the value returned by s_getpc
2042 // (which is the address of the s_add_u32 instruction) to the global
2043 // variable, but since the encoding of $symbol starts 4 bytes after the start
2044 // of the s_add_u32 instruction, we end up with an offset that is 4 bytes too
2045 // small. This requires us to add 4 to the global variable offset in order to
2046 // compute the correct address.
2048 LLT ConstPtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
2050 Register PCReg = PtrTy.getSizeInBits() != 32 ? DstReg :
2051 B.getMRI()->createGenericVirtualRegister(ConstPtrTy);
2053 MachineInstrBuilder MIB = B.buildInstr(AMDGPU::SI_PC_ADD_REL_OFFSET)
2056 MIB.addGlobalAddress(GV, Offset + 4, GAFlags);
2057 if (GAFlags == SIInstrInfo::MO_NONE)
2060 MIB.addGlobalAddress(GV, Offset + 4, GAFlags + 1);
2062 B.getMRI()->setRegClass(PCReg, &AMDGPU::SReg_64RegClass);
2064 if (PtrTy.getSizeInBits() == 32)
2065 B.buildExtract(DstReg, PCReg, 0);
2069 bool AMDGPULegalizerInfo::legalizeGlobalValue(
2070 MachineInstr &MI, MachineRegisterInfo &MRI,
2071 MachineIRBuilder &B) const {
2072 Register DstReg = MI.getOperand(0).getReg();
2073 LLT Ty = MRI.getType(DstReg);
2074 unsigned AS = Ty.getAddressSpace();
2076 const GlobalValue *GV = MI.getOperand(1).getGlobal();
2077 MachineFunction &MF = B.getMF();
2078 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2080 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
2081 if (!MFI->isEntryFunction()) {
2082 const Function &Fn = MF.getFunction();
2083 DiagnosticInfoUnsupported BadLDSDecl(
2084 Fn, "local memory global used by non-kernel function", MI.getDebugLoc(),
2086 Fn.getContext().diagnose(BadLDSDecl);
2088 // We currently don't have a way to correctly allocate LDS objects that
2089 // aren't directly associated with a kernel. We do force inlining of
2090 // functions that use local objects. However, if these dead functions are
2091 // not eliminated, we don't want a compile time error. Just emit a warning
2092 // and a trap, since there should be no callable path here.
2093 B.buildIntrinsic(Intrinsic::trap, ArrayRef<Register>(), true);
2094 B.buildUndef(DstReg);
2095 MI.eraseFromParent();
2099 // TODO: We could emit code to handle the initialization somewhere.
2100 if (!AMDGPUTargetLowering::hasDefinedInitializer(GV)) {
2101 const SITargetLowering *TLI = ST.getTargetLowering();
2102 if (!TLI->shouldUseLDSConstAddress(GV)) {
2103 MI.getOperand(1).setTargetFlags(SIInstrInfo::MO_ABS32_LO);
2104 return true; // Leave in place;
2109 MFI->allocateLDSGlobal(B.getDataLayout(), *cast<GlobalVariable>(GV)));
2110 MI.eraseFromParent();
2114 const Function &Fn = MF.getFunction();
2115 DiagnosticInfoUnsupported BadInit(
2116 Fn, "unsupported initializer for address space", MI.getDebugLoc());
2117 Fn.getContext().diagnose(BadInit);
2121 const SITargetLowering *TLI = ST.getTargetLowering();
2123 if (TLI->shouldEmitFixup(GV)) {
2124 buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0);
2125 MI.eraseFromParent();
2129 if (TLI->shouldEmitPCReloc(GV)) {
2130 buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0, SIInstrInfo::MO_REL32);
2131 MI.eraseFromParent();
2135 LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
2136 Register GOTAddr = MRI.createGenericVirtualRegister(PtrTy);
2138 MachineMemOperand *GOTMMO = MF.getMachineMemOperand(
2139 MachinePointerInfo::getGOT(MF),
2140 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
2141 MachineMemOperand::MOInvariant,
2142 8 /*Size*/, Align(8));
2144 buildPCRelGlobalAddress(GOTAddr, PtrTy, B, GV, 0, SIInstrInfo::MO_GOTPCREL32);
2146 if (Ty.getSizeInBits() == 32) {
2147 // Truncate if this is a 32-bit constant adrdess.
2148 auto Load = B.buildLoad(PtrTy, GOTAddr, *GOTMMO);
2149 B.buildExtract(DstReg, Load, 0);
2151 B.buildLoad(DstReg, GOTAddr, *GOTMMO);
2153 MI.eraseFromParent();
2157 bool AMDGPULegalizerInfo::legalizeLoad(
2158 MachineInstr &MI, MachineRegisterInfo &MRI,
2159 MachineIRBuilder &B, GISelChangeObserver &Observer) const {
2160 LLT ConstPtr = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
2161 auto Cast = B.buildAddrSpaceCast(ConstPtr, MI.getOperand(1).getReg());
2162 Observer.changingInstr(MI);
2163 MI.getOperand(1).setReg(Cast.getReg(0));
2164 Observer.changedInstr(MI);
2168 bool AMDGPULegalizerInfo::legalizeFMad(
2169 MachineInstr &MI, MachineRegisterInfo &MRI,
2170 MachineIRBuilder &B) const {
2171 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
2172 assert(Ty.isScalar());
2174 MachineFunction &MF = B.getMF();
2175 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2177 // TODO: Always legal with future ftz flag.
2178 // FIXME: Do we need just output?
2179 if (Ty == LLT::scalar(32) && !MFI->getMode().allFP32Denormals())
2181 if (Ty == LLT::scalar(16) && !MFI->getMode().allFP64FP16Denormals())
2184 MachineIRBuilder HelperBuilder(MI);
2185 GISelObserverWrapper DummyObserver;
2186 LegalizerHelper Helper(MF, DummyObserver, HelperBuilder);
2187 return Helper.lowerFMad(MI) == LegalizerHelper::Legalized;
2190 bool AMDGPULegalizerInfo::legalizeAtomicCmpXChg(
2191 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
2192 Register DstReg = MI.getOperand(0).getReg();
2193 Register PtrReg = MI.getOperand(1).getReg();
2194 Register CmpVal = MI.getOperand(2).getReg();
2195 Register NewVal = MI.getOperand(3).getReg();
2197 assert(SITargetLowering::isFlatGlobalAddrSpace(
2198 MRI.getType(PtrReg).getAddressSpace()) &&
2199 "this should not have been custom lowered");
2201 LLT ValTy = MRI.getType(CmpVal);
2202 LLT VecTy = LLT::vector(2, ValTy);
2204 Register PackedVal = B.buildBuildVector(VecTy, { NewVal, CmpVal }).getReg(0);
2206 B.buildInstr(AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG)
2210 .setMemRefs(MI.memoperands());
2212 MI.eraseFromParent();
2216 bool AMDGPULegalizerInfo::legalizeFlog(
2217 MachineInstr &MI, MachineIRBuilder &B, double Log2BaseInverted) const {
2218 Register Dst = MI.getOperand(0).getReg();
2219 Register Src = MI.getOperand(1).getReg();
2220 LLT Ty = B.getMRI()->getType(Dst);
2221 unsigned Flags = MI.getFlags();
2223 auto Log2Operand = B.buildFLog2(Ty, Src, Flags);
2224 auto Log2BaseInvertedOperand = B.buildFConstant(Ty, Log2BaseInverted);
2226 B.buildFMul(Dst, Log2Operand, Log2BaseInvertedOperand, Flags);
2227 MI.eraseFromParent();
2231 bool AMDGPULegalizerInfo::legalizeFExp(MachineInstr &MI,
2232 MachineIRBuilder &B) const {
2233 Register Dst = MI.getOperand(0).getReg();
2234 Register Src = MI.getOperand(1).getReg();
2235 unsigned Flags = MI.getFlags();
2236 LLT Ty = B.getMRI()->getType(Dst);
2238 auto K = B.buildFConstant(Ty, numbers::log2e);
2239 auto Mul = B.buildFMul(Ty, Src, K, Flags);
2240 B.buildFExp2(Dst, Mul, Flags);
2241 MI.eraseFromParent();
2245 bool AMDGPULegalizerInfo::legalizeFPow(MachineInstr &MI,
2246 MachineIRBuilder &B) const {
2247 Register Dst = MI.getOperand(0).getReg();
2248 Register Src0 = MI.getOperand(1).getReg();
2249 Register Src1 = MI.getOperand(2).getReg();
2250 unsigned Flags = MI.getFlags();
2251 LLT Ty = B.getMRI()->getType(Dst);
2252 const LLT S16 = LLT::scalar(16);
2253 const LLT S32 = LLT::scalar(32);
2256 auto Log = B.buildFLog2(S32, Src0, Flags);
2257 auto Mul = B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {S32}, false)
2258 .addUse(Log.getReg(0))
2261 B.buildFExp2(Dst, Mul, Flags);
2262 } else if (Ty == S16) {
2263 // There's no f16 fmul_legacy, so we need to convert for it.
2264 auto Log = B.buildFLog2(S16, Src0, Flags);
2265 auto Ext0 = B.buildFPExt(S32, Log, Flags);
2266 auto Ext1 = B.buildFPExt(S32, Src1, Flags);
2267 auto Mul = B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {S32}, false)
2268 .addUse(Ext0.getReg(0))
2269 .addUse(Ext1.getReg(0))
2272 B.buildFExp2(Dst, B.buildFPTrunc(S16, Mul), Flags);
2276 MI.eraseFromParent();
2280 // Find a source register, ignoring any possible source modifiers.
2281 static Register stripAnySourceMods(Register OrigSrc, MachineRegisterInfo &MRI) {
2282 Register ModSrc = OrigSrc;
2283 if (MachineInstr *SrcFNeg = getOpcodeDef(AMDGPU::G_FNEG, ModSrc, MRI)) {
2284 ModSrc = SrcFNeg->getOperand(1).getReg();
2285 if (MachineInstr *SrcFAbs = getOpcodeDef(AMDGPU::G_FABS, ModSrc, MRI))
2286 ModSrc = SrcFAbs->getOperand(1).getReg();
2287 } else if (MachineInstr *SrcFAbs = getOpcodeDef(AMDGPU::G_FABS, ModSrc, MRI))
2288 ModSrc = SrcFAbs->getOperand(1).getReg();
2292 bool AMDGPULegalizerInfo::legalizeFFloor(MachineInstr &MI,
2293 MachineRegisterInfo &MRI,
2294 MachineIRBuilder &B) const {
2296 const LLT S1 = LLT::scalar(1);
2297 const LLT S64 = LLT::scalar(64);
2298 Register Dst = MI.getOperand(0).getReg();
2299 Register OrigSrc = MI.getOperand(1).getReg();
2300 unsigned Flags = MI.getFlags();
2301 assert(ST.hasFractBug() && MRI.getType(Dst) == S64 &&
2302 "this should not have been custom lowered");
2304 // V_FRACT is buggy on SI, so the F32 version is never used and (x-floor(x))
2305 // is used instead. However, SI doesn't have V_FLOOR_F64, so the most
2306 // efficient way to implement it is using V_FRACT_F64. The workaround for the
2308 // fract(x) = isnan(x) ? x : min(V_FRACT(x), 0.99999999999999999)
2310 // Convert floor(x) to (x - fract(x))
2312 auto Fract = B.buildIntrinsic(Intrinsic::amdgcn_fract, {S64}, false)
2316 // Give source modifier matching some assistance before obscuring a foldable
2319 // TODO: We can avoid the neg on the fract? The input sign to fract
2320 // shouldn't matter?
2321 Register ModSrc = stripAnySourceMods(OrigSrc, MRI);
2323 auto Const = B.buildFConstant(S64, BitsToDouble(0x3fefffffffffffff));
2325 Register Min = MRI.createGenericVirtualRegister(S64);
2327 // We don't need to concern ourselves with the snan handling difference, so
2328 // use the one which will directly select.
2329 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
2330 if (MFI->getMode().IEEE)
2331 B.buildFMinNumIEEE(Min, Fract, Const, Flags);
2333 B.buildFMinNum(Min, Fract, Const, Flags);
2335 Register CorrectedFract = Min;
2336 if (!MI.getFlag(MachineInstr::FmNoNans)) {
2337 auto IsNan = B.buildFCmp(CmpInst::FCMP_ORD, S1, ModSrc, ModSrc, Flags);
2338 CorrectedFract = B.buildSelect(S64, IsNan, ModSrc, Min, Flags).getReg(0);
2341 auto NegFract = B.buildFNeg(S64, CorrectedFract, Flags);
2342 B.buildFAdd(Dst, OrigSrc, NegFract, Flags);
2344 MI.eraseFromParent();
2348 // Turn an illegal packed v2s16 build vector into bit operations.
2349 // TODO: This should probably be a bitcast action in LegalizerHelper.
2350 bool AMDGPULegalizerInfo::legalizeBuildVector(
2351 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
2352 Register Dst = MI.getOperand(0).getReg();
2353 const LLT S32 = LLT::scalar(32);
2354 assert(MRI.getType(Dst) == LLT::vector(2, 16));
2356 Register Src0 = MI.getOperand(1).getReg();
2357 Register Src1 = MI.getOperand(2).getReg();
2358 assert(MRI.getType(Src0) == LLT::scalar(16));
2360 auto Merge = B.buildMerge(S32, {Src0, Src1});
2361 B.buildBitcast(Dst, Merge);
2363 MI.eraseFromParent();
2367 // Return the use branch instruction, otherwise null if the usage is invalid.
2368 static MachineInstr *verifyCFIntrinsic(MachineInstr &MI,
2369 MachineRegisterInfo &MRI,
2371 MachineBasicBlock *&UncondBrTarget) {
2372 Register CondDef = MI.getOperand(0).getReg();
2373 if (!MRI.hasOneNonDBGUse(CondDef))
2376 MachineBasicBlock *Parent = MI.getParent();
2377 MachineInstr &UseMI = *MRI.use_instr_nodbg_begin(CondDef);
2378 if (UseMI.getParent() != Parent ||
2379 UseMI.getOpcode() != AMDGPU::G_BRCOND)
2382 // Make sure the cond br is followed by a G_BR, or is the last instruction.
2383 MachineBasicBlock::iterator Next = std::next(UseMI.getIterator());
2384 if (Next == Parent->end()) {
2385 MachineFunction::iterator NextMBB = std::next(Parent->getIterator());
2386 if (NextMBB == Parent->getParent()->end()) // Illegal intrinsic use.
2388 UncondBrTarget = &*NextMBB;
2390 if (Next->getOpcode() != AMDGPU::G_BR)
2393 UncondBrTarget = Br->getOperand(0).getMBB();
2399 Register AMDGPULegalizerInfo::insertLiveInCopy(MachineIRBuilder &B,
2400 MachineRegisterInfo &MRI,
2402 Register PhyReg) const {
2403 assert(PhyReg.isPhysical() && "Physical register expected");
2405 // Insert the live-in copy, if required, by defining destination virtual
2407 // FIXME: It seems EmitLiveInCopies isn't called anywhere?
2408 if (!MRI.getVRegDef(LiveIn)) {
2409 // FIXME: Should have scoped insert pt
2410 MachineBasicBlock &OrigInsBB = B.getMBB();
2411 auto OrigInsPt = B.getInsertPt();
2413 MachineBasicBlock &EntryMBB = B.getMF().front();
2414 EntryMBB.addLiveIn(PhyReg);
2415 B.setInsertPt(EntryMBB, EntryMBB.begin());
2416 B.buildCopy(LiveIn, PhyReg);
2418 B.setInsertPt(OrigInsBB, OrigInsPt);
2424 Register AMDGPULegalizerInfo::getLiveInRegister(MachineIRBuilder &B,
2425 MachineRegisterInfo &MRI,
2426 Register PhyReg, LLT Ty,
2427 bool InsertLiveInCopy) const {
2428 assert(PhyReg.isPhysical() && "Physical register expected");
2430 // Get or create virtual live-in regester
2431 Register LiveIn = MRI.getLiveInVirtReg(PhyReg);
2433 LiveIn = MRI.createGenericVirtualRegister(Ty);
2434 MRI.addLiveIn(PhyReg, LiveIn);
2437 // When the actual true copy required is from virtual register to physical
2438 // register (to be inserted later), live-in copy insertion from physical
2439 // to register virtual register is not required
2440 if (!InsertLiveInCopy)
2443 return insertLiveInCopy(B, MRI, LiveIn, PhyReg);
2446 const ArgDescriptor *AMDGPULegalizerInfo::getArgDescriptor(
2447 MachineIRBuilder &B, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
2448 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
2449 const ArgDescriptor *Arg;
2450 const TargetRegisterClass *RC;
2452 std::tie(Arg, RC, ArgTy) = MFI->getPreloadedValue(ArgType);
2454 LLVM_DEBUG(dbgs() << "Required arg register missing\n");
2460 bool AMDGPULegalizerInfo::loadInputValue(Register DstReg, MachineIRBuilder &B,
2461 const ArgDescriptor *Arg) const {
2462 if (!Arg->isRegister() || !Arg->getRegister().isValid())
2463 return false; // TODO: Handle these
2465 Register SrcReg = Arg->getRegister();
2466 assert(SrcReg.isPhysical() && "Physical register expected");
2467 assert(DstReg.isVirtual() && "Virtual register expected");
2469 MachineRegisterInfo &MRI = *B.getMRI();
2471 LLT Ty = MRI.getType(DstReg);
2472 Register LiveIn = getLiveInRegister(B, MRI, SrcReg, Ty);
2474 if (Arg->isMasked()) {
2475 // TODO: Should we try to emit this once in the entry block?
2476 const LLT S32 = LLT::scalar(32);
2477 const unsigned Mask = Arg->getMask();
2478 const unsigned Shift = countTrailingZeros<unsigned>(Mask);
2480 Register AndMaskSrc = LiveIn;
2483 auto ShiftAmt = B.buildConstant(S32, Shift);
2484 AndMaskSrc = B.buildLShr(S32, LiveIn, ShiftAmt).getReg(0);
2487 B.buildAnd(DstReg, AndMaskSrc, B.buildConstant(S32, Mask >> Shift));
2489 B.buildCopy(DstReg, LiveIn);
2495 bool AMDGPULegalizerInfo::legalizePreloadedArgIntrin(
2496 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
2497 AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
2499 const ArgDescriptor *Arg = getArgDescriptor(B, ArgType);
2503 if (!loadInputValue(MI.getOperand(0).getReg(), B, Arg))
2506 MI.eraseFromParent();
2510 bool AMDGPULegalizerInfo::legalizeFDIV(MachineInstr &MI,
2511 MachineRegisterInfo &MRI,
2512 MachineIRBuilder &B) const {
2513 Register Dst = MI.getOperand(0).getReg();
2514 LLT DstTy = MRI.getType(Dst);
2515 LLT S16 = LLT::scalar(16);
2516 LLT S32 = LLT::scalar(32);
2517 LLT S64 = LLT::scalar(64);
2519 if (legalizeFastUnsafeFDIV(MI, MRI, B))
2523 return legalizeFDIV16(MI, MRI, B);
2525 return legalizeFDIV32(MI, MRI, B);
2527 return legalizeFDIV64(MI, MRI, B);
2532 void AMDGPULegalizerInfo::legalizeUDIV_UREM32Impl(MachineIRBuilder &B,
2537 const LLT S1 = LLT::scalar(1);
2538 const LLT S32 = LLT::scalar(32);
2540 // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
2541 // algorithm used here.
2543 // Initial estimate of inv(y).
2544 auto FloatY = B.buildUITOFP(S32, Y);
2545 auto RcpIFlag = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {S32}, {FloatY});
2546 auto Scale = B.buildFConstant(S32, BitsToFloat(0x4f7ffffe));
2547 auto ScaledY = B.buildFMul(S32, RcpIFlag, Scale);
2548 auto Z = B.buildFPTOUI(S32, ScaledY);
2550 // One round of UNR.
2551 auto NegY = B.buildSub(S32, B.buildConstant(S32, 0), Y);
2552 auto NegYZ = B.buildMul(S32, NegY, Z);
2553 Z = B.buildAdd(S32, Z, B.buildUMulH(S32, Z, NegYZ));
2555 // Quotient/remainder estimate.
2556 auto Q = B.buildUMulH(S32, X, Z);
2557 auto R = B.buildSub(S32, X, B.buildMul(S32, Q, Y));
2559 // First quotient/remainder refinement.
2560 auto One = B.buildConstant(S32, 1);
2561 auto Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
2563 Q = B.buildSelect(S32, Cond, B.buildAdd(S32, Q, One), Q);
2564 R = B.buildSelect(S32, Cond, B.buildSub(S32, R, Y), R);
2566 // Second quotient/remainder refinement.
2567 Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
2569 B.buildSelect(DstReg, Cond, B.buildAdd(S32, Q, One), Q);
2571 B.buildSelect(DstReg, Cond, B.buildSub(S32, R, Y), R);
2574 bool AMDGPULegalizerInfo::legalizeUDIV_UREM32(MachineInstr &MI,
2575 MachineRegisterInfo &MRI,
2576 MachineIRBuilder &B) const {
2577 const bool IsDiv = MI.getOpcode() == AMDGPU::G_UDIV;
2578 Register DstReg = MI.getOperand(0).getReg();
2579 Register Num = MI.getOperand(1).getReg();
2580 Register Den = MI.getOperand(2).getReg();
2581 legalizeUDIV_UREM32Impl(B, DstReg, Num, Den, IsDiv);
2582 MI.eraseFromParent();
2586 // Build integer reciprocal sequence arounud V_RCP_IFLAG_F32
2588 // Return lo, hi of result
2590 // %cvt.lo = G_UITOFP Val.lo
2591 // %cvt.hi = G_UITOFP Val.hi
2592 // %mad = G_FMAD %cvt.hi, 2**32, %cvt.lo
2593 // %rcp = G_AMDGPU_RCP_IFLAG %mad
2594 // %mul1 = G_FMUL %rcp, 0x5f7ffffc
2595 // %mul2 = G_FMUL %mul1, 2**(-32)
2596 // %trunc = G_INTRINSIC_TRUNC %mul2
2597 // %mad2 = G_FMAD %trunc, -(2**32), %mul1
2598 // return {G_FPTOUI %mad2, G_FPTOUI %trunc}
2599 static std::pair<Register, Register> emitReciprocalU64(MachineIRBuilder &B,
2601 const LLT S32 = LLT::scalar(32);
2602 auto Unmerge = B.buildUnmerge(S32, Val);
2604 auto CvtLo = B.buildUITOFP(S32, Unmerge.getReg(0));
2605 auto CvtHi = B.buildUITOFP(S32, Unmerge.getReg(1));
2607 auto Mad = B.buildFMAD(S32, CvtHi, // 2**32
2608 B.buildFConstant(S32, BitsToFloat(0x4f800000)), CvtLo);
2610 auto Rcp = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {S32}, {Mad});
2612 B.buildFMul(S32, Rcp, B.buildFConstant(S32, BitsToFloat(0x5f7ffffc)));
2616 B.buildFMul(S32, Mul1, B.buildFConstant(S32, BitsToFloat(0x2f800000)));
2617 auto Trunc = B.buildIntrinsicTrunc(S32, Mul2);
2620 auto Mad2 = B.buildFMAD(S32, Trunc,
2621 B.buildFConstant(S32, BitsToFloat(0xcf800000)), Mul1);
2623 auto ResultLo = B.buildFPTOUI(S32, Mad2);
2624 auto ResultHi = B.buildFPTOUI(S32, Trunc);
2626 return {ResultLo.getReg(0), ResultHi.getReg(0)};
2629 void AMDGPULegalizerInfo::legalizeUDIV_UREM64Impl(MachineIRBuilder &B,
2634 const LLT S32 = LLT::scalar(32);
2635 const LLT S64 = LLT::scalar(64);
2636 const LLT S1 = LLT::scalar(1);
2637 Register RcpLo, RcpHi;
2639 std::tie(RcpLo, RcpHi) = emitReciprocalU64(B, Denom);
2641 auto Rcp = B.buildMerge(S64, {RcpLo, RcpHi});
2643 auto Zero64 = B.buildConstant(S64, 0);
2644 auto NegDenom = B.buildSub(S64, Zero64, Denom);
2646 auto MulLo1 = B.buildMul(S64, NegDenom, Rcp);
2647 auto MulHi1 = B.buildUMulH(S64, Rcp, MulLo1);
2649 auto UnmergeMulHi1 = B.buildUnmerge(S32, MulHi1);
2650 Register MulHi1_Lo = UnmergeMulHi1.getReg(0);
2651 Register MulHi1_Hi = UnmergeMulHi1.getReg(1);
2653 auto Add1_Lo = B.buildUAddo(S32, S1, RcpLo, MulHi1_Lo);
2654 auto Add1_Hi = B.buildUAdde(S32, S1, RcpHi, MulHi1_Hi, Add1_Lo.getReg(1));
2655 auto Add1_HiNc = B.buildAdd(S32, RcpHi, MulHi1_Hi);
2656 auto Add1 = B.buildMerge(S64, {Add1_Lo, Add1_Hi});
2658 auto MulLo2 = B.buildMul(S64, NegDenom, Add1);
2659 auto MulHi2 = B.buildUMulH(S64, Add1, MulLo2);
2660 auto UnmergeMulHi2 = B.buildUnmerge(S32, MulHi2);
2661 Register MulHi2_Lo = UnmergeMulHi2.getReg(0);
2662 Register MulHi2_Hi = UnmergeMulHi2.getReg(1);
2664 auto Zero32 = B.buildConstant(S32, 0);
2665 auto Add2_Lo = B.buildUAddo(S32, S1, Add1_Lo, MulHi2_Lo);
2667 B.buildUAdde(S32, S1, Add1_HiNc, MulHi2_Hi, Add1_Lo.getReg(1));
2668 auto Add2_Hi = B.buildUAdde(S32, S1, Add2_HiC, Zero32, Add2_Lo.getReg(1));
2669 auto Add2 = B.buildMerge(S64, {Add2_Lo, Add2_Hi});
2671 auto UnmergeNumer = B.buildUnmerge(S32, Numer);
2672 Register NumerLo = UnmergeNumer.getReg(0);
2673 Register NumerHi = UnmergeNumer.getReg(1);
2675 auto MulHi3 = B.buildUMulH(S64, Numer, Add2);
2676 auto Mul3 = B.buildMul(S64, Denom, MulHi3);
2677 auto UnmergeMul3 = B.buildUnmerge(S32, Mul3);
2678 Register Mul3_Lo = UnmergeMul3.getReg(0);
2679 Register Mul3_Hi = UnmergeMul3.getReg(1);
2680 auto Sub1_Lo = B.buildUSubo(S32, S1, NumerLo, Mul3_Lo);
2681 auto Sub1_Hi = B.buildUSube(S32, S1, NumerHi, Mul3_Hi, Sub1_Lo.getReg(1));
2682 auto Sub1_Mi = B.buildSub(S32, NumerHi, Mul3_Hi);
2683 auto Sub1 = B.buildMerge(S64, {Sub1_Lo, Sub1_Hi});
2685 auto UnmergeDenom = B.buildUnmerge(S32, Denom);
2686 Register DenomLo = UnmergeDenom.getReg(0);
2687 Register DenomHi = UnmergeDenom.getReg(1);
2689 auto CmpHi = B.buildICmp(CmpInst::ICMP_UGE, S1, Sub1_Hi, DenomHi);
2690 auto C1 = B.buildSExt(S32, CmpHi);
2692 auto CmpLo = B.buildICmp(CmpInst::ICMP_UGE, S1, Sub1_Lo, DenomLo);
2693 auto C2 = B.buildSExt(S32, CmpLo);
2695 auto CmpEq = B.buildICmp(CmpInst::ICMP_EQ, S1, Sub1_Hi, DenomHi);
2696 auto C3 = B.buildSelect(S32, CmpEq, C2, C1);
2698 // TODO: Here and below portions of the code can be enclosed into if/endif.
2699 // Currently control flow is unconditional and we have 4 selects after
2700 // potential endif to substitute PHIs.
2703 auto Sub2_Lo = B.buildUSubo(S32, S1, Sub1_Lo, DenomLo);
2704 auto Sub2_Mi = B.buildUSube(S32, S1, Sub1_Mi, DenomHi, Sub1_Lo.getReg(1));
2705 auto Sub2_Hi = B.buildUSube(S32, S1, Sub2_Mi, Zero32, Sub2_Lo.getReg(1));
2706 auto Sub2 = B.buildMerge(S64, {Sub2_Lo, Sub2_Hi});
2708 auto One64 = B.buildConstant(S64, 1);
2709 auto Add3 = B.buildAdd(S64, MulHi3, One64);
2712 B.buildSExt(S32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Hi, DenomHi));
2714 B.buildSExt(S32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Lo, DenomLo));
2715 auto C6 = B.buildSelect(
2716 S32, B.buildICmp(CmpInst::ICMP_EQ, S1, Sub2_Hi, DenomHi), C5, C4);
2719 auto Add4 = B.buildAdd(S64, Add3, One64);
2720 auto Sub3_Lo = B.buildUSubo(S32, S1, Sub2_Lo, DenomLo);
2722 auto Sub3_Mi = B.buildUSube(S32, S1, Sub2_Mi, DenomHi, Sub2_Lo.getReg(1));
2723 auto Sub3_Hi = B.buildUSube(S32, S1, Sub3_Mi, Zero32, Sub3_Lo.getReg(1));
2724 auto Sub3 = B.buildMerge(S64, {Sub3_Lo, Sub3_Hi});
2730 auto Sel1 = B.buildSelect(
2731 S64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Add4, Add3);
2732 B.buildSelect(DstReg,
2733 B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32), Sel1, MulHi3);
2735 auto Sel2 = B.buildSelect(
2736 S64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Sub3, Sub2);
2737 B.buildSelect(DstReg,
2738 B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32), Sel2, Sub1);
2742 bool AMDGPULegalizerInfo::legalizeUDIV_UREM(MachineInstr &MI,
2743 MachineRegisterInfo &MRI,
2744 MachineIRBuilder &B) const {
2745 const LLT S64 = LLT::scalar(64);
2746 const LLT S32 = LLT::scalar(32);
2747 const bool IsDiv = MI.getOpcode() == AMDGPU::G_UDIV;
2748 Register DstReg = MI.getOperand(0).getReg();
2749 Register Num = MI.getOperand(1).getReg();
2750 Register Den = MI.getOperand(2).getReg();
2751 LLT Ty = MRI.getType(DstReg);
2754 legalizeUDIV_UREM32Impl(B, DstReg, Num, Den, IsDiv);
2756 legalizeUDIV_UREM64Impl(B, DstReg, Num, Den, IsDiv);
2760 MI.eraseFromParent();
2765 bool AMDGPULegalizerInfo::legalizeSDIV_SREM(MachineInstr &MI,
2766 MachineRegisterInfo &MRI,
2767 MachineIRBuilder &B) const {
2768 const LLT S64 = LLT::scalar(64);
2769 const LLT S32 = LLT::scalar(32);
2771 Register DstReg = MI.getOperand(0).getReg();
2772 const LLT Ty = MRI.getType(DstReg);
2773 if (Ty != S32 && Ty != S64)
2776 const bool IsDiv = MI.getOpcode() == AMDGPU::G_SDIV;
2778 Register LHS = MI.getOperand(1).getReg();
2779 Register RHS = MI.getOperand(2).getReg();
2781 auto SignBitOffset = B.buildConstant(S32, Ty.getSizeInBits() - 1);
2782 auto LHSign = B.buildAShr(Ty, LHS, SignBitOffset);
2783 auto RHSign = B.buildAShr(Ty, RHS, SignBitOffset);
2785 LHS = B.buildAdd(Ty, LHS, LHSign).getReg(0);
2786 RHS = B.buildAdd(Ty, RHS, RHSign).getReg(0);
2788 LHS = B.buildXor(Ty, LHS, LHSign).getReg(0);
2789 RHS = B.buildXor(Ty, RHS, RHSign).getReg(0);
2791 Register UDivRem = MRI.createGenericVirtualRegister(Ty);
2793 legalizeUDIV_UREM32Impl(B, UDivRem, LHS, RHS, IsDiv);
2795 legalizeUDIV_UREM64Impl(B, UDivRem, LHS, RHS, IsDiv);
2799 Sign = B.buildXor(Ty, LHSign, RHSign).getReg(0);
2801 Sign = LHSign.getReg(0); // Remainder sign is the same as LHS
2803 UDivRem = B.buildXor(Ty, UDivRem, Sign).getReg(0);
2804 B.buildSub(DstReg, UDivRem, Sign);
2806 MI.eraseFromParent();
2810 bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV(MachineInstr &MI,
2811 MachineRegisterInfo &MRI,
2812 MachineIRBuilder &B) const {
2813 Register Res = MI.getOperand(0).getReg();
2814 Register LHS = MI.getOperand(1).getReg();
2815 Register RHS = MI.getOperand(2).getReg();
2817 uint16_t Flags = MI.getFlags();
2819 LLT ResTy = MRI.getType(Res);
2820 LLT S32 = LLT::scalar(32);
2821 LLT S64 = LLT::scalar(64);
2823 const MachineFunction &MF = B.getMF();
2825 MF.getTarget().Options.UnsafeFPMath || MI.getFlag(MachineInstr::FmArcp);
2827 if (!MF.getTarget().Options.UnsafeFPMath && ResTy == S64)
2830 if (!Unsafe && ResTy == S32 &&
2831 MF.getInfo<SIMachineFunctionInfo>()->getMode().allFP32Denormals())
2834 if (auto CLHS = getConstantFPVRegVal(LHS, MRI)) {
2836 if (CLHS->isExactlyValue(1.0)) {
2837 B.buildIntrinsic(Intrinsic::amdgcn_rcp, Res, false)
2841 MI.eraseFromParent();
2845 // -1 / x -> RCP( FNEG(x) )
2846 if (CLHS->isExactlyValue(-1.0)) {
2847 auto FNeg = B.buildFNeg(ResTy, RHS, Flags);
2848 B.buildIntrinsic(Intrinsic::amdgcn_rcp, Res, false)
2849 .addUse(FNeg.getReg(0))
2852 MI.eraseFromParent();
2857 // x / y -> x * (1.0 / y)
2859 auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy}, false)
2862 B.buildFMul(Res, LHS, RCP, Flags);
2864 MI.eraseFromParent();
2871 bool AMDGPULegalizerInfo::legalizeFDIV16(MachineInstr &MI,
2872 MachineRegisterInfo &MRI,
2873 MachineIRBuilder &B) const {
2874 Register Res = MI.getOperand(0).getReg();
2875 Register LHS = MI.getOperand(1).getReg();
2876 Register RHS = MI.getOperand(2).getReg();
2878 uint16_t Flags = MI.getFlags();
2880 LLT S16 = LLT::scalar(16);
2881 LLT S32 = LLT::scalar(32);
2883 auto LHSExt = B.buildFPExt(S32, LHS, Flags);
2884 auto RHSExt = B.buildFPExt(S32, RHS, Flags);
2886 auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32}, false)
2887 .addUse(RHSExt.getReg(0))
2890 auto QUOT = B.buildFMul(S32, LHSExt, RCP, Flags);
2891 auto RDst = B.buildFPTrunc(S16, QUOT, Flags);
2893 B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res, false)
2894 .addUse(RDst.getReg(0))
2899 MI.eraseFromParent();
2903 // Enable or disable FP32 denorm mode. When 'Enable' is true, emit instructions
2904 // to enable denorm mode. When 'Enable' is false, disable denorm mode.
2905 static void toggleSPDenormMode(bool Enable,
2906 MachineIRBuilder &B,
2907 const GCNSubtarget &ST,
2908 AMDGPU::SIModeRegisterDefaults Mode) {
2909 // Set SP denorm mode to this value.
2910 unsigned SPDenormMode =
2911 Enable ? FP_DENORM_FLUSH_NONE : Mode.fpDenormModeSPValue();
2913 if (ST.hasDenormModeInst()) {
2914 // Preserve default FP64FP16 denorm mode while updating FP32 mode.
2915 uint32_t DPDenormModeDefault = Mode.fpDenormModeDPValue();
2917 uint32_t NewDenormModeValue = SPDenormMode | (DPDenormModeDefault << 2);
2918 B.buildInstr(AMDGPU::S_DENORM_MODE)
2919 .addImm(NewDenormModeValue);
2922 // Select FP32 bit field in mode register.
2923 unsigned SPDenormModeBitField = AMDGPU::Hwreg::ID_MODE |
2924 (4 << AMDGPU::Hwreg::OFFSET_SHIFT_) |
2925 (1 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_);
2927 B.buildInstr(AMDGPU::S_SETREG_IMM32_B32)
2928 .addImm(SPDenormMode)
2929 .addImm(SPDenormModeBitField);
2933 bool AMDGPULegalizerInfo::legalizeFDIV32(MachineInstr &MI,
2934 MachineRegisterInfo &MRI,
2935 MachineIRBuilder &B) const {
2936 Register Res = MI.getOperand(0).getReg();
2937 Register LHS = MI.getOperand(1).getReg();
2938 Register RHS = MI.getOperand(2).getReg();
2939 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
2940 AMDGPU::SIModeRegisterDefaults Mode = MFI->getMode();
2942 uint16_t Flags = MI.getFlags();
2944 LLT S32 = LLT::scalar(32);
2945 LLT S1 = LLT::scalar(1);
2947 auto One = B.buildFConstant(S32, 1.0f);
2949 auto DenominatorScaled =
2950 B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S32, S1}, false)
2955 auto NumeratorScaled =
2956 B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S32, S1}, false)
2962 auto ApproxRcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32}, false)
2963 .addUse(DenominatorScaled.getReg(0))
2965 auto NegDivScale0 = B.buildFNeg(S32, DenominatorScaled, Flags);
2967 // FIXME: Doesn't correctly model the FP mode switch, and the FP operations
2968 // aren't modeled as reading it.
2969 if (!Mode.allFP32Denormals())
2970 toggleSPDenormMode(true, B, ST, Mode);
2972 auto Fma0 = B.buildFMA(S32, NegDivScale0, ApproxRcp, One, Flags);
2973 auto Fma1 = B.buildFMA(S32, Fma0, ApproxRcp, ApproxRcp, Flags);
2974 auto Mul = B.buildFMul(S32, NumeratorScaled, Fma1, Flags);
2975 auto Fma2 = B.buildFMA(S32, NegDivScale0, Mul, NumeratorScaled, Flags);
2976 auto Fma3 = B.buildFMA(S32, Fma2, Fma1, Mul, Flags);
2977 auto Fma4 = B.buildFMA(S32, NegDivScale0, Fma3, NumeratorScaled, Flags);
2979 if (!Mode.allFP32Denormals())
2980 toggleSPDenormMode(false, B, ST, Mode);
2982 auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {S32}, false)
2983 .addUse(Fma4.getReg(0))
2984 .addUse(Fma1.getReg(0))
2985 .addUse(Fma3.getReg(0))
2986 .addUse(NumeratorScaled.getReg(1))
2989 B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res, false)
2990 .addUse(Fmas.getReg(0))
2995 MI.eraseFromParent();
2999 bool AMDGPULegalizerInfo::legalizeFDIV64(MachineInstr &MI,
3000 MachineRegisterInfo &MRI,
3001 MachineIRBuilder &B) const {
3002 Register Res = MI.getOperand(0).getReg();
3003 Register LHS = MI.getOperand(1).getReg();
3004 Register RHS = MI.getOperand(2).getReg();
3006 uint16_t Flags = MI.getFlags();
3008 LLT S64 = LLT::scalar(64);
3009 LLT S1 = LLT::scalar(1);
3011 auto One = B.buildFConstant(S64, 1.0);
3013 auto DivScale0 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S64, S1}, false)
3019 auto NegDivScale0 = B.buildFNeg(S64, DivScale0.getReg(0), Flags);
3021 auto Rcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S64}, false)
3022 .addUse(DivScale0.getReg(0))
3025 auto Fma0 = B.buildFMA(S64, NegDivScale0, Rcp, One, Flags);
3026 auto Fma1 = B.buildFMA(S64, Rcp, Fma0, Rcp, Flags);
3027 auto Fma2 = B.buildFMA(S64, NegDivScale0, Fma1, One, Flags);
3029 auto DivScale1 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S64, S1}, false)
3035 auto Fma3 = B.buildFMA(S64, Fma1, Fma2, Fma1, Flags);
3036 auto Mul = B.buildFMul(S64, DivScale1.getReg(0), Fma3, Flags);
3037 auto Fma4 = B.buildFMA(S64, NegDivScale0, Mul, DivScale1.getReg(0), Flags);
3040 if (!ST.hasUsableDivScaleConditionOutput()) {
3041 // Workaround a hardware bug on SI where the condition output from div_scale
3044 LLT S32 = LLT::scalar(32);
3046 auto NumUnmerge = B.buildUnmerge(S32, LHS);
3047 auto DenUnmerge = B.buildUnmerge(S32, RHS);
3048 auto Scale0Unmerge = B.buildUnmerge(S32, DivScale0);
3049 auto Scale1Unmerge = B.buildUnmerge(S32, DivScale1);
3051 auto CmpNum = B.buildICmp(ICmpInst::ICMP_EQ, S1, NumUnmerge.getReg(1),
3052 Scale1Unmerge.getReg(1));
3053 auto CmpDen = B.buildICmp(ICmpInst::ICMP_EQ, S1, DenUnmerge.getReg(1),
3054 Scale0Unmerge.getReg(1));
3055 Scale = B.buildXor(S1, CmpNum, CmpDen).getReg(0);
3057 Scale = DivScale1.getReg(1);
3060 auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {S64}, false)
3061 .addUse(Fma4.getReg(0))
3062 .addUse(Fma3.getReg(0))
3063 .addUse(Mul.getReg(0))
3067 B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, makeArrayRef(Res), false)
3068 .addUse(Fmas.getReg(0))
3073 MI.eraseFromParent();
3077 bool AMDGPULegalizerInfo::legalizeFDIVFastIntrin(MachineInstr &MI,
3078 MachineRegisterInfo &MRI,
3079 MachineIRBuilder &B) const {
3080 Register Res = MI.getOperand(0).getReg();
3081 Register LHS = MI.getOperand(2).getReg();
3082 Register RHS = MI.getOperand(3).getReg();
3083 uint16_t Flags = MI.getFlags();
3085 LLT S32 = LLT::scalar(32);
3086 LLT S1 = LLT::scalar(1);
3088 auto Abs = B.buildFAbs(S32, RHS, Flags);
3089 const APFloat C0Val(1.0f);
3091 auto C0 = B.buildConstant(S32, 0x6f800000);
3092 auto C1 = B.buildConstant(S32, 0x2f800000);
3093 auto C2 = B.buildConstant(S32, FloatToBits(1.0f));
3095 auto CmpRes = B.buildFCmp(CmpInst::FCMP_OGT, S1, Abs, C0, Flags);
3096 auto Sel = B.buildSelect(S32, CmpRes, C1, C2, Flags);
3098 auto Mul0 = B.buildFMul(S32, RHS, Sel, Flags);
3100 auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32}, false)
3101 .addUse(Mul0.getReg(0))
3104 auto Mul1 = B.buildFMul(S32, LHS, RCP, Flags);
3106 B.buildFMul(Res, Sel, Mul1, Flags);
3108 MI.eraseFromParent();
3112 bool AMDGPULegalizerInfo::legalizeImplicitArgPtr(MachineInstr &MI,
3113 MachineRegisterInfo &MRI,
3114 MachineIRBuilder &B) const {
3115 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
3116 if (!MFI->isEntryFunction()) {
3117 return legalizePreloadedArgIntrin(MI, MRI, B,
3118 AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
3122 ST.getTargetLowering()->getImplicitParameterOffset(
3123 B.getMF(), AMDGPUTargetLowering::FIRST_IMPLICIT);
3124 Register DstReg = MI.getOperand(0).getReg();
3125 LLT DstTy = MRI.getType(DstReg);
3126 LLT IdxTy = LLT::scalar(DstTy.getSizeInBits());
3128 const ArgDescriptor *Arg;
3129 const TargetRegisterClass *RC;
3131 std::tie(Arg, RC, ArgTy) =
3132 MFI->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
3136 Register KernargPtrReg = MRI.createGenericVirtualRegister(DstTy);
3137 if (!loadInputValue(KernargPtrReg, B, Arg))
3140 B.buildPtrAdd(DstReg, KernargPtrReg, B.buildConstant(IdxTy, Offset).getReg(0));
3141 MI.eraseFromParent();
3145 bool AMDGPULegalizerInfo::legalizeIsAddrSpace(MachineInstr &MI,
3146 MachineRegisterInfo &MRI,
3147 MachineIRBuilder &B,
3148 unsigned AddrSpace) const {
3149 Register ApertureReg = getSegmentAperture(AddrSpace, MRI, B);
3150 auto Hi32 = B.buildExtract(LLT::scalar(32), MI.getOperand(2).getReg(), 32);
3151 B.buildICmp(ICmpInst::ICMP_EQ, MI.getOperand(0), Hi32, ApertureReg);
3152 MI.eraseFromParent();
3156 // The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
3157 // offset (the offset that is included in bounds checking and swizzling, to be
3158 // split between the instruction's voffset and immoffset fields) and soffset
3159 // (the offset that is excluded from bounds checking and swizzling, to go in
3160 // the instruction's soffset field). This function takes the first kind of
3161 // offset and figures out how to split it between voffset and immoffset.
3162 std::tuple<Register, unsigned, unsigned>
3163 AMDGPULegalizerInfo::splitBufferOffsets(MachineIRBuilder &B,
3164 Register OrigOffset) const {
3165 const unsigned MaxImm = 4095;
3167 unsigned TotalConstOffset;
3168 MachineInstr *OffsetDef;
3169 const LLT S32 = LLT::scalar(32);
3171 std::tie(BaseReg, TotalConstOffset, OffsetDef)
3172 = AMDGPU::getBaseWithConstantOffset(*B.getMRI(), OrigOffset);
3174 unsigned ImmOffset = TotalConstOffset;
3176 // If the immediate value is too big for the immoffset field, put the value
3177 // and -4096 into the immoffset field so that the value that is copied/added
3178 // for the voffset field is a multiple of 4096, and it stands more chance
3179 // of being CSEd with the copy/add for another similar load/store.
3180 // However, do not do that rounding down to a multiple of 4096 if that is a
3181 // negative number, as it appears to be illegal to have a negative offset
3182 // in the vgpr, even if adding the immediate offset makes it positive.
3183 unsigned Overflow = ImmOffset & ~MaxImm;
3184 ImmOffset -= Overflow;
3185 if ((int32_t)Overflow < 0) {
3186 Overflow += ImmOffset;
3190 if (Overflow != 0) {
3192 BaseReg = B.buildConstant(S32, Overflow).getReg(0);
3194 auto OverflowVal = B.buildConstant(S32, Overflow);
3195 BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0);
3200 BaseReg = B.buildConstant(S32, 0).getReg(0);
3202 return std::make_tuple(BaseReg, ImmOffset, TotalConstOffset);
3205 /// Handle register layout difference for f16 images for some subtargets.
3206 Register AMDGPULegalizerInfo::handleD16VData(MachineIRBuilder &B,
3207 MachineRegisterInfo &MRI,
3208 Register Reg) const {
3209 if (!ST.hasUnpackedD16VMem())
3212 const LLT S16 = LLT::scalar(16);
3213 const LLT S32 = LLT::scalar(32);
3214 LLT StoreVT = MRI.getType(Reg);
3215 assert(StoreVT.isVector() && StoreVT.getElementType() == S16);
3217 auto Unmerge = B.buildUnmerge(S16, Reg);
3219 SmallVector<Register, 4> WideRegs;
3220 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
3221 WideRegs.push_back(B.buildAnyExt(S32, Unmerge.getReg(I)).getReg(0));
3223 int NumElts = StoreVT.getNumElements();
3225 return B.buildBuildVector(LLT::vector(NumElts, S32), WideRegs).getReg(0);
3228 Register AMDGPULegalizerInfo::fixStoreSourceType(
3229 MachineIRBuilder &B, Register VData, bool IsFormat) const {
3230 MachineRegisterInfo *MRI = B.getMRI();
3231 LLT Ty = MRI->getType(VData);
3233 const LLT S16 = LLT::scalar(16);
3235 // Fixup illegal register types for i8 stores.
3236 if (Ty == LLT::scalar(8) || Ty == S16) {
3237 Register AnyExt = B.buildAnyExt(LLT::scalar(32), VData).getReg(0);
3241 if (Ty.isVector()) {
3242 if (Ty.getElementType() == S16 && Ty.getNumElements() <= 4) {
3244 return handleD16VData(B, *MRI, VData);
3251 bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
3252 MachineRegisterInfo &MRI,
3253 MachineIRBuilder &B,
3255 bool IsFormat) const {
3256 Register VData = MI.getOperand(1).getReg();
3257 LLT Ty = MRI.getType(VData);
3258 LLT EltTy = Ty.getScalarType();
3259 const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
3260 const LLT S32 = LLT::scalar(32);
3262 VData = fixStoreSourceType(B, VData, IsFormat);
3263 Register RSrc = MI.getOperand(2).getReg();
3265 MachineMemOperand *MMO = *MI.memoperands_begin();
3266 const int MemSize = MMO->getSize();
3269 unsigned TotalOffset;
3271 // The typed intrinsics add an immediate after the registers.
3272 const unsigned NumVIndexOps = IsTyped ? 8 : 7;
3274 // The struct intrinsic variants add one additional operand over raw.
3275 const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
3279 VIndex = MI.getOperand(3).getReg();
3283 Register VOffset = MI.getOperand(3 + OpOffset).getReg();
3284 Register SOffset = MI.getOperand(4 + OpOffset).getReg();
3286 unsigned Format = 0;
3288 Format = MI.getOperand(5 + OpOffset).getImm();
3292 unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();
3294 std::tie(VOffset, ImmOffset, TotalOffset) = splitBufferOffsets(B, VOffset);
3295 if (TotalOffset != 0)
3296 MMO = B.getMF().getMachineMemOperand(MMO, TotalOffset, MemSize);
3300 Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16 :
3301 AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT;
3302 } else if (IsFormat) {
3303 Opc = IsD16 ? AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16 :
3304 AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT;
3308 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE;
3311 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT;
3314 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE;
3320 VIndex = B.buildConstant(S32, 0).getReg(0);
3322 auto MIB = B.buildInstr(Opc)
3323 .addUse(VData) // vdata
3324 .addUse(RSrc) // rsrc
3325 .addUse(VIndex) // vindex
3326 .addUse(VOffset) // voffset
3327 .addUse(SOffset) // soffset
3328 .addImm(ImmOffset); // offset(imm)
3333 MIB.addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm)
3334 .addImm(HasVIndex ? -1 : 0) // idxen(imm)
3335 .addMemOperand(MMO);
3337 MI.eraseFromParent();
3341 bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
3342 MachineRegisterInfo &MRI,
3343 MachineIRBuilder &B,
3345 bool IsTyped) const {
3346 // FIXME: Verifier should enforce 1 MMO for these intrinsics.
3347 MachineMemOperand *MMO = *MI.memoperands_begin();
3348 const int MemSize = MMO->getSize();
3349 const LLT S32 = LLT::scalar(32);
3351 Register Dst = MI.getOperand(0).getReg();
3352 Register RSrc = MI.getOperand(2).getReg();
3354 // The typed intrinsics add an immediate after the registers.
3355 const unsigned NumVIndexOps = IsTyped ? 8 : 7;
3357 // The struct intrinsic variants add one additional operand over raw.
3358 const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
3362 VIndex = MI.getOperand(3).getReg();
3366 Register VOffset = MI.getOperand(3 + OpOffset).getReg();
3367 Register SOffset = MI.getOperand(4 + OpOffset).getReg();
3369 unsigned Format = 0;
3371 Format = MI.getOperand(5 + OpOffset).getImm();
3375 unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();
3377 unsigned TotalOffset;
3379 LLT Ty = MRI.getType(Dst);
3380 LLT EltTy = Ty.getScalarType();
3381 const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
3382 const bool Unpacked = ST.hasUnpackedD16VMem();
3384 std::tie(VOffset, ImmOffset, TotalOffset) = splitBufferOffsets(B, VOffset);
3385 if (TotalOffset != 0)
3386 MMO = B.getMF().getMachineMemOperand(MMO, TotalOffset, MemSize);
3391 Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 :
3392 AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT;
3393 } else if (IsFormat) {
3394 Opc = IsD16 ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16 :
3395 AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT;
3399 Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE;
3402 Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT;
3405 Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD;
3410 Register LoadDstReg;
3412 bool IsExtLoad = (!IsD16 && MemSize < 4) || (IsD16 && !Ty.isVector());
3413 LLT UnpackedTy = Ty.changeElementSize(32);
3416 LoadDstReg = B.getMRI()->createGenericVirtualRegister(S32);
3417 else if (Unpacked && IsD16 && Ty.isVector())
3418 LoadDstReg = B.getMRI()->createGenericVirtualRegister(UnpackedTy);
3423 VIndex = B.buildConstant(S32, 0).getReg(0);
3425 auto MIB = B.buildInstr(Opc)
3426 .addDef(LoadDstReg) // vdata
3427 .addUse(RSrc) // rsrc
3428 .addUse(VIndex) // vindex
3429 .addUse(VOffset) // voffset
3430 .addUse(SOffset) // soffset
3431 .addImm(ImmOffset); // offset(imm)
3436 MIB.addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm)
3437 .addImm(HasVIndex ? -1 : 0) // idxen(imm)
3438 .addMemOperand(MMO);
3440 if (LoadDstReg != Dst) {
3441 B.setInsertPt(B.getMBB(), ++B.getInsertPt());
3443 // Widen result for extending loads was widened.
3445 B.buildTrunc(Dst, LoadDstReg);
3447 // Repack to original 16-bit vector result
3448 // FIXME: G_TRUNC should work, but legalization currently fails
3449 auto Unmerge = B.buildUnmerge(S32, LoadDstReg);
3450 SmallVector<Register, 4> Repack;
3451 for (unsigned I = 0, N = Unmerge->getNumOperands() - 1; I != N; ++I)
3452 Repack.push_back(B.buildTrunc(EltTy, Unmerge.getReg(I)).getReg(0));
3453 B.buildMerge(Dst, Repack);
3457 MI.eraseFromParent();
3461 bool AMDGPULegalizerInfo::legalizeAtomicIncDec(MachineInstr &MI,
3462 MachineIRBuilder &B,
3464 unsigned Opc = IsInc ? AMDGPU::G_AMDGPU_ATOMIC_INC :
3465 AMDGPU::G_AMDGPU_ATOMIC_DEC;
3467 .addDef(MI.getOperand(0).getReg())
3468 .addUse(MI.getOperand(2).getReg())
3469 .addUse(MI.getOperand(3).getReg())
3471 MI.eraseFromParent();
3475 static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID) {
3477 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
3478 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
3479 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP;
3480 case Intrinsic::amdgcn_raw_buffer_atomic_add:
3481 case Intrinsic::amdgcn_struct_buffer_atomic_add:
3482 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD;
3483 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
3484 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
3485 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB;
3486 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
3487 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
3488 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN;
3489 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
3490 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
3491 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN;
3492 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
3493 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
3494 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX;
3495 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
3496 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
3497 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX;
3498 case Intrinsic::amdgcn_raw_buffer_atomic_and:
3499 case Intrinsic::amdgcn_struct_buffer_atomic_and:
3500 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND;
3501 case Intrinsic::amdgcn_raw_buffer_atomic_or:
3502 case Intrinsic::amdgcn_struct_buffer_atomic_or:
3503 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR;
3504 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
3505 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
3506 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR;
3507 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
3508 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
3509 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC;
3510 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
3511 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
3512 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC;
3513 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
3514 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
3515 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP;
3517 llvm_unreachable("unhandled atomic opcode");
3521 bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
3522 MachineIRBuilder &B,
3523 Intrinsic::ID IID) const {
3524 const bool IsCmpSwap = IID == Intrinsic::amdgcn_raw_buffer_atomic_cmpswap ||
3525 IID == Intrinsic::amdgcn_struct_buffer_atomic_cmpswap;
3527 Register Dst = MI.getOperand(0).getReg();
3528 Register VData = MI.getOperand(2).getReg();
3534 CmpVal = MI.getOperand(3 + OpOffset).getReg();
3538 Register RSrc = MI.getOperand(3 + OpOffset).getReg();
3539 const unsigned NumVIndexOps = IsCmpSwap ? 9 : 8;
3541 // The struct intrinsic variants add one additional operand over raw.
3542 const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
3545 VIndex = MI.getOperand(4 + OpOffset).getReg();
3549 Register VOffset = MI.getOperand(4 + OpOffset).getReg();
3550 Register SOffset = MI.getOperand(5 + OpOffset).getReg();
3551 unsigned AuxiliaryData = MI.getOperand(6 + OpOffset).getImm();
3553 MachineMemOperand *MMO = *MI.memoperands_begin();
3556 unsigned TotalOffset;
3557 std::tie(VOffset, ImmOffset, TotalOffset) = splitBufferOffsets(B, VOffset);
3558 if (TotalOffset != 0)
3559 MMO = B.getMF().getMachineMemOperand(MMO, TotalOffset, MMO->getSize());
3562 VIndex = B.buildConstant(LLT::scalar(32), 0).getReg(0);
3564 auto MIB = B.buildInstr(getBufferAtomicPseudo(IID))
3566 .addUse(VData); // vdata
3571 MIB.addUse(RSrc) // rsrc
3572 .addUse(VIndex) // vindex
3573 .addUse(VOffset) // voffset
3574 .addUse(SOffset) // soffset
3575 .addImm(ImmOffset) // offset(imm)
3576 .addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm)
3577 .addImm(HasVIndex ? -1 : 0) // idxen(imm)
3578 .addMemOperand(MMO);
3580 MI.eraseFromParent();
3584 /// Turn a set of s16 typed registers in \p A16AddrRegs into a dword sized
3585 /// vector with s16 typed elements.
3586 static void packImageA16AddressToDwords(MachineIRBuilder &B, MachineInstr &MI,
3587 SmallVectorImpl<Register> &PackedAddrs,
3588 int AddrIdx, int DimIdx, int EndIdx,
3590 const LLT S16 = LLT::scalar(16);
3591 const LLT V2S16 = LLT::vector(2, 16);
3593 for (int I = AddrIdx; I < EndIdx; ++I) {
3594 MachineOperand &SrcOp = MI.getOperand(I);
3596 continue; // _L to _LZ may have eliminated this.
3598 Register AddrReg = SrcOp.getReg();
3601 AddrReg = B.buildBitcast(V2S16, AddrReg).getReg(0);
3602 PackedAddrs.push_back(AddrReg);
3604 // Dz/dh, dz/dv and the last odd coord are packed with undef. Also, in 1D,
3605 // derivatives dx/dh and dx/dv are packed with undef.
3606 if (((I + 1) >= EndIdx) ||
3607 ((NumGradients / 2) % 2 == 1 &&
3608 (I == DimIdx + (NumGradients / 2) - 1 ||
3609 I == DimIdx + NumGradients - 1)) ||
3610 // Check for _L to _LZ optimization
3611 !MI.getOperand(I + 1).isReg()) {
3612 PackedAddrs.push_back(
3613 B.buildBuildVector(V2S16, {AddrReg, B.buildUndef(S16).getReg(0)})
3616 PackedAddrs.push_back(
3617 B.buildBuildVector(V2S16, {AddrReg, MI.getOperand(I + 1).getReg()})
3625 /// Convert from separate vaddr components to a single vector address register,
3626 /// and replace the remaining operands with $noreg.
3627 static void convertImageAddrToPacked(MachineIRBuilder &B, MachineInstr &MI,
3628 int DimIdx, int NumVAddrs) {
3629 const LLT S32 = LLT::scalar(32);
3631 SmallVector<Register, 8> AddrRegs;
3632 for (int I = 0; I != NumVAddrs; ++I) {
3633 MachineOperand &SrcOp = MI.getOperand(DimIdx + I);
3634 if (SrcOp.isReg()) {
3635 AddrRegs.push_back(SrcOp.getReg());
3636 assert(B.getMRI()->getType(SrcOp.getReg()) == S32);
3640 int NumAddrRegs = AddrRegs.size();
3641 if (NumAddrRegs != 1) {
3642 // Round up to 8 elements for v5-v7
3643 // FIXME: Missing intermediate sized register classes and instructions.
3644 if (NumAddrRegs > 4 && !isPowerOf2_32(NumAddrRegs)) {
3645 const int RoundedNumRegs = NextPowerOf2(NumAddrRegs);
3646 auto Undef = B.buildUndef(S32);
3647 AddrRegs.append(RoundedNumRegs - NumAddrRegs, Undef.getReg(0));
3648 NumAddrRegs = RoundedNumRegs;
3651 auto VAddr = B.buildBuildVector(LLT::vector(NumAddrRegs, 32), AddrRegs);
3652 MI.getOperand(DimIdx).setReg(VAddr.getReg(0));
3655 for (int I = 1; I != NumVAddrs; ++I) {
3656 MachineOperand &SrcOp = MI.getOperand(DimIdx + I);
3658 MI.getOperand(DimIdx + I).setReg(AMDGPU::NoRegister);
3662 /// Rewrite image intrinsics to use register layouts expected by the subtarget.
3664 /// Depending on the subtarget, load/store with 16-bit element data need to be
3665 /// rewritten to use the low half of 32-bit registers, or directly use a packed
3666 /// layout. 16-bit addresses should also sometimes be packed into 32-bit
3669 /// We don't want to directly select image instructions just yet, but also want
3670 /// to exposes all register repacking to the legalizer/combiners. We also don't
3671 /// want a selected instrution entering RegBankSelect. In order to avoid
3672 /// defining a multitude of intermediate image instructions, directly hack on
3673 /// the intrinsic's arguments. In cases like a16 addreses, this requires padding
3674 /// now unnecessary arguments with $noreg.
3675 bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
3676 MachineInstr &MI, MachineIRBuilder &B,
3677 GISelChangeObserver &Observer,
3678 const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr) const {
3680 const int NumDefs = MI.getNumExplicitDefs();
3681 bool IsTFE = NumDefs == 2;
3682 // We are only processing the operands of d16 image operations on subtargets
3683 // that use the unpacked register layout, or need to repack the TFE result.
3685 // TODO: Do we need to guard against already legalized intrinsics?
3686 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
3687 AMDGPU::getMIMGBaseOpcodeInfo(ImageDimIntr->BaseOpcode);
3689 MachineRegisterInfo *MRI = B.getMRI();
3690 const LLT S32 = LLT::scalar(32);
3691 const LLT S16 = LLT::scalar(16);
3692 const LLT V2S16 = LLT::vector(2, 16);
3694 // Index of first address argument
3695 const int AddrIdx = getImageVAddrIdxBegin(BaseOpcode, NumDefs);
3697 int NumVAddrs, NumGradients;
3698 std::tie(NumVAddrs, NumGradients) = getImageNumVAddr(ImageDimIntr, BaseOpcode);
3699 const int DMaskIdx = BaseOpcode->Atomic ? -1 :
3700 getDMaskIdx(BaseOpcode, NumDefs);
3703 // Check for 16 bit addresses and pack if true.
3704 int DimIdx = AddrIdx + BaseOpcode->NumExtraArgs;
3705 LLT GradTy = MRI->getType(MI.getOperand(DimIdx).getReg());
3706 LLT AddrTy = MRI->getType(MI.getOperand(DimIdx + NumGradients).getReg());
3707 const bool IsG16 = GradTy == S16;
3708 const bool IsA16 = AddrTy == S16;
3711 if (!BaseOpcode->Atomic) {
3712 DMask = MI.getOperand(DMaskIdx).getImm();
3713 if (BaseOpcode->Gather4) {
3715 } else if (DMask != 0) {
3716 DMaskLanes = countPopulation(DMask);
3717 } else if (!IsTFE && !BaseOpcode->Store) {
3718 // If dmask is 0, this is a no-op load. This can be eliminated.
3719 B.buildUndef(MI.getOperand(0));
3720 MI.eraseFromParent();
3725 Observer.changingInstr(MI);
3726 auto ChangedInstr = make_scope_exit([&] { Observer.changedInstr(MI); });
3728 unsigned NewOpcode = NumDefs == 0 ?
3729 AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE : AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD;
3731 // Track that we legalized this
3732 MI.setDesc(B.getTII().get(NewOpcode));
3734 // Expecting to get an error flag since TFC is on - and dmask is 0 Force
3735 // dmask to be at least 1 otherwise the instruction will fail
3736 if (IsTFE && DMask == 0) {
3739 MI.getOperand(DMaskIdx).setImm(DMask);
3742 if (BaseOpcode->Atomic) {
3743 Register VData0 = MI.getOperand(2).getReg();
3744 LLT Ty = MRI->getType(VData0);
3746 // TODO: Allow atomic swap and bit ops for v2s16/v4s16
3750 if (BaseOpcode->AtomicX2) {
3751 Register VData1 = MI.getOperand(3).getReg();
3752 // The two values are packed in one register.
3753 LLT PackedTy = LLT::vector(2, Ty);
3754 auto Concat = B.buildBuildVector(PackedTy, {VData0, VData1});
3755 MI.getOperand(2).setReg(Concat.getReg(0));
3756 MI.getOperand(3).setReg(AMDGPU::NoRegister);
3760 int CorrectedNumVAddrs = NumVAddrs;
3762 // Optimize _L to _LZ when _L is zero
3763 if (const AMDGPU::MIMGLZMappingInfo *LZMappingInfo =
3764 AMDGPU::getMIMGLZMappingInfo(ImageDimIntr->BaseOpcode)) {
3765 const ConstantFP *ConstantLod;
3766 const int LodIdx = AddrIdx + NumVAddrs - 1;
3768 if (mi_match(MI.getOperand(LodIdx).getReg(), *MRI, m_GFCst(ConstantLod))) {
3769 if (ConstantLod->isZero() || ConstantLod->isNegative()) {
3770 // Set new opcode to _lz variant of _l, and change the intrinsic ID.
3771 ImageDimIntr = AMDGPU::getImageDimInstrinsicByBaseOpcode(
3772 LZMappingInfo->LZ, ImageDimIntr->Dim);
3774 // The starting indexes should remain in the same place.
3776 --CorrectedNumVAddrs;
3778 MI.getOperand(MI.getNumExplicitDefs()).setIntrinsicID(
3779 static_cast<Intrinsic::ID>(ImageDimIntr->Intr));
3780 MI.RemoveOperand(LodIdx);
3785 // Optimize _mip away, when 'lod' is zero
3786 if (AMDGPU::getMIMGMIPMappingInfo(ImageDimIntr->BaseOpcode)) {
3787 int64_t ConstantLod;
3788 const int LodIdx = AddrIdx + NumVAddrs - 1;
3790 if (mi_match(MI.getOperand(LodIdx).getReg(), *MRI, m_ICst(ConstantLod))) {
3791 if (ConstantLod == 0) {
3792 // TODO: Change intrinsic opcode and remove operand instead or replacing
3793 // it with 0, as the _L to _LZ handling is done above.
3794 MI.getOperand(LodIdx).ChangeToImmediate(0);
3795 --CorrectedNumVAddrs;
3800 // Rewrite the addressing register layout before doing anything else.
3801 if (IsA16 || IsG16) {
3803 // Target must support the feature and gradients need to be 16 bit too
3804 if (!ST.hasA16() || !IsG16)
3806 } else if (!ST.hasG16())
3809 if (NumVAddrs > 1) {
3810 SmallVector<Register, 4> PackedRegs;
3811 // Don't compress addresses for G16
3812 const int PackEndIdx =
3813 IsA16 ? (AddrIdx + NumVAddrs) : (DimIdx + NumGradients);
3814 packImageA16AddressToDwords(B, MI, PackedRegs, AddrIdx, DimIdx,
3815 PackEndIdx, NumGradients);
3818 // Add uncompressed address
3819 for (int I = DimIdx + NumGradients; I != AddrIdx + NumVAddrs; ++I) {
3820 int AddrReg = MI.getOperand(I).getReg();
3821 assert(B.getMRI()->getType(AddrReg) == LLT::scalar(32));
3822 PackedRegs.push_back(AddrReg);
3826 // See also below in the non-a16 branch
3827 const bool UseNSA = PackedRegs.size() >= 3 && ST.hasNSAEncoding();
3829 if (!UseNSA && PackedRegs.size() > 1) {
3830 LLT PackedAddrTy = LLT::vector(2 * PackedRegs.size(), 16);
3831 auto Concat = B.buildConcatVectors(PackedAddrTy, PackedRegs);
3832 PackedRegs[0] = Concat.getReg(0);
3833 PackedRegs.resize(1);
3836 const int NumPacked = PackedRegs.size();
3837 for (int I = 0; I != NumVAddrs; ++I) {
3838 MachineOperand &SrcOp = MI.getOperand(AddrIdx + I);
3839 if (!SrcOp.isReg()) {
3840 assert(SrcOp.isImm() && SrcOp.getImm() == 0);
3844 assert(SrcOp.getReg() != AMDGPU::NoRegister);
3847 SrcOp.setReg(PackedRegs[I]);
3849 SrcOp.setReg(AMDGPU::NoRegister);
3853 // If the register allocator cannot place the address registers contiguously
3854 // without introducing moves, then using the non-sequential address encoding
3855 // is always preferable, since it saves VALU instructions and is usually a
3856 // wash in terms of code size or even better.
3858 // However, we currently have no way of hinting to the register allocator
3859 // that MIMG addresses should be placed contiguously when it is possible to
3860 // do so, so force non-NSA for the common 2-address case as a heuristic.
3862 // SIShrinkInstructions will convert NSA encodings to non-NSA after register
3863 // allocation when possible.
3864 const bool UseNSA = CorrectedNumVAddrs >= 3 && ST.hasNSAEncoding();
3866 if (!UseNSA && NumVAddrs > 1)
3867 convertImageAddrToPacked(B, MI, AddrIdx, NumVAddrs);
3875 MI.addOperand(MachineOperand::CreateImm(Flags));
3877 if (BaseOpcode->Store) { // No TFE for stores?
3878 // TODO: Handle dmask trim
3879 Register VData = MI.getOperand(1).getReg();
3880 LLT Ty = MRI->getType(VData);
3881 if (!Ty.isVector() || Ty.getElementType() != S16)
3884 Register RepackedReg = handleD16VData(B, *MRI, VData);
3885 if (RepackedReg != VData) {
3886 MI.getOperand(1).setReg(RepackedReg);
3892 Register DstReg = MI.getOperand(0).getReg();
3893 LLT Ty = MRI->getType(DstReg);
3894 const LLT EltTy = Ty.getScalarType();
3895 const bool IsD16 = Ty.getScalarType() == S16;
3896 const int NumElts = Ty.isVector() ? Ty.getNumElements() : 1;
3898 // Confirm that the return type is large enough for the dmask specified
3899 if (NumElts < DMaskLanes)
3902 if (NumElts > 4 || DMaskLanes > 4)
3905 const unsigned AdjustedNumElts = DMaskLanes == 0 ? 1 : DMaskLanes;
3906 const LLT AdjustedTy = Ty.changeNumElements(AdjustedNumElts);
3908 // The raw dword aligned data component of the load. The only legal cases
3909 // where this matters should be when using the packed D16 format, for
3910 // s16 -> <2 x s16>, and <3 x s16> -> <4 x s16>,
3913 // S32 vector to to cover all data, plus TFE result element.
3916 // Register type to use for each loaded component. Will be S32 or V2S16.
3919 if (IsD16 && ST.hasUnpackedD16VMem()) {
3920 RoundedTy = LLT::scalarOrVector(AdjustedNumElts, 32);
3921 TFETy = LLT::vector(AdjustedNumElts + 1, 32);
3924 unsigned EltSize = EltTy.getSizeInBits();
3925 unsigned RoundedElts = (AdjustedTy.getSizeInBits() + 31) / 32;
3926 unsigned RoundedSize = 32 * RoundedElts;
3927 RoundedTy = LLT::scalarOrVector(RoundedSize / EltSize, EltSize);
3928 TFETy = LLT::vector(RoundedSize / 32 + 1, S32);
3929 RegTy = !IsTFE && EltSize == 16 ? V2S16 : S32;
3932 // The return type does not need adjustment.
3933 // TODO: Should we change s16 case to s32 or <2 x s16>?
3934 if (!IsTFE && (RoundedTy == Ty || !Ty.isVector()))
3939 // Insert after the instruction.
3940 B.setInsertPt(*MI.getParent(), ++MI.getIterator());
3942 // TODO: For TFE with d16, if we used a TFE type that was a multiple of <2 x
3943 // s16> instead of s32, we would only need 1 bitcast instead of multiple.
3944 const LLT LoadResultTy = IsTFE ? TFETy : RoundedTy;
3945 const int ResultNumRegs = LoadResultTy.getSizeInBits() / 32;
3947 Register NewResultReg = MRI->createGenericVirtualRegister(LoadResultTy);
3949 MI.getOperand(0).setReg(NewResultReg);
3951 // In the IR, TFE is supposed to be used with a 2 element struct return
3952 // type. The intruction really returns these two values in one contiguous
3953 // register, with one additional dword beyond the loaded data. Rewrite the
3954 // return type to use a single register result.
3957 Dst1Reg = MI.getOperand(1).getReg();
3958 if (MRI->getType(Dst1Reg) != S32)
3961 // TODO: Make sure the TFE operand bit is set.
3962 MI.RemoveOperand(1);
3964 // Handle the easy case that requires no repack instructions.
3966 B.buildUnmerge({DstReg, Dst1Reg}, NewResultReg);
3971 // Now figure out how to copy the new result register back into the old
3973 SmallVector<Register, 5> ResultRegs(ResultNumRegs, Dst1Reg);
3975 const int NumDataRegs = IsTFE ? ResultNumRegs - 1 : ResultNumRegs;
3977 if (ResultNumRegs == 1) {
3979 ResultRegs[0] = NewResultReg;
3981 // We have to repack into a new vector of some kind.
3982 for (int I = 0; I != NumDataRegs; ++I)
3983 ResultRegs[I] = MRI->createGenericVirtualRegister(RegTy);
3984 B.buildUnmerge(ResultRegs, NewResultReg);
3986 // Drop the final TFE element to get the data part. The TFE result is
3987 // directly written to the right place already.
3989 ResultRegs.resize(NumDataRegs);
3992 // For an s16 scalar result, we form an s32 result with a truncate regardless
3993 // of packed vs. unpacked.
3994 if (IsD16 && !Ty.isVector()) {
3995 B.buildTrunc(DstReg, ResultRegs[0]);
3999 // Avoid a build/concat_vector of 1 entry.
4000 if (Ty == V2S16 && NumDataRegs == 1 && !ST.hasUnpackedD16VMem()) {
4001 B.buildBitcast(DstReg, ResultRegs[0]);
4005 assert(Ty.isVector());
4008 // For packed D16 results with TFE enabled, all the data components are
4009 // S32. Cast back to the expected type.
4011 // TODO: We don't really need to use load s32 elements. We would only need one
4012 // cast for the TFE result if a multiple of v2s16 was used.
4013 if (RegTy != V2S16 && !ST.hasUnpackedD16VMem()) {
4014 for (Register &Reg : ResultRegs)
4015 Reg = B.buildBitcast(V2S16, Reg).getReg(0);
4016 } else if (ST.hasUnpackedD16VMem()) {
4017 for (Register &Reg : ResultRegs)
4018 Reg = B.buildTrunc(S16, Reg).getReg(0);
4022 auto padWithUndef = [&](LLT Ty, int NumElts) {
4025 Register Undef = B.buildUndef(Ty).getReg(0);
4026 for (int I = 0; I != NumElts; ++I)
4027 ResultRegs.push_back(Undef);
4030 // Pad out any elements eliminated due to the dmask.
4031 LLT ResTy = MRI->getType(ResultRegs[0]);
4032 if (!ResTy.isVector()) {
4033 padWithUndef(ResTy, NumElts - ResultRegs.size());
4034 B.buildBuildVector(DstReg, ResultRegs);
4038 assert(!ST.hasUnpackedD16VMem() && ResTy == V2S16);
4039 const int RegsToCover = (Ty.getSizeInBits() + 31) / 32;
4041 // Deal with the one annoying legal case.
4042 const LLT V3S16 = LLT::vector(3, 16);
4044 padWithUndef(ResTy, RegsToCover - ResultRegs.size() + 1);
4045 auto Concat = B.buildConcatVectors(LLT::vector(6, 16), ResultRegs);
4046 B.buildUnmerge({DstReg, MRI->createGenericVirtualRegister(V3S16)}, Concat);
4050 padWithUndef(ResTy, RegsToCover - ResultRegs.size());
4051 B.buildConcatVectors(DstReg, ResultRegs);
4055 bool AMDGPULegalizerInfo::legalizeSBufferLoad(
4056 MachineInstr &MI, MachineIRBuilder &B,
4057 GISelChangeObserver &Observer) const {
4058 Register Dst = MI.getOperand(0).getReg();
4059 LLT Ty = B.getMRI()->getType(Dst);
4060 unsigned Size = Ty.getSizeInBits();
4061 MachineFunction &MF = B.getMF();
4063 Observer.changingInstr(MI);
4065 // FIXME: We don't really need this intermediate instruction. The intrinsic
4066 // should be fixed to have a memory operand. Since it's readnone, we're not
4067 // allowed to add one.
4068 MI.setDesc(B.getTII().get(AMDGPU::G_AMDGPU_S_BUFFER_LOAD));
4069 MI.RemoveOperand(1); // Remove intrinsic ID
4071 // FIXME: When intrinsic definition is fixed, this should have an MMO already.
4072 // TODO: Should this use datalayout alignment?
4073 const unsigned MemSize = (Size + 7) / 8;
4074 const Align MemAlign(4);
4075 MachineMemOperand *MMO = MF.getMachineMemOperand(
4076 MachinePointerInfo(),
4077 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
4078 MachineMemOperand::MOInvariant,
4080 MI.addMemOperand(MF, MMO);
4082 // There are no 96-bit result scalar loads, but widening to 128-bit should
4083 // always be legal. We may need to restore this to a 96-bit result if it turns
4084 // out this needs to be converted to a vector load during RegBankSelect.
4085 if (!isPowerOf2_32(Size)) {
4086 LegalizerHelper Helper(MF, *this, Observer, B);
4089 Helper.moreElementsVectorDst(MI, getPow2VectorType(Ty), 0);
4091 Helper.widenScalarDst(MI, getPow2ScalarType(Ty), 0);
4094 Observer.changedInstr(MI);
4098 bool AMDGPULegalizerInfo::legalizeTrapIntrinsic(MachineInstr &MI,
4099 MachineRegisterInfo &MRI,
4100 MachineIRBuilder &B) const {
4101 // Is non-HSA path or trap-handler disabled? then, insert s_endpgm instruction
4102 if (ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbiHsa ||
4103 !ST.isTrapHandlerEnabled()) {
4104 B.buildInstr(AMDGPU::S_ENDPGM).addImm(0);
4106 // Pass queue pointer to trap handler as input, and insert trap instruction
4107 // Reference: https://llvm.org/docs/AMDGPUUsage.html#trap-handler-abi
4108 const ArgDescriptor *Arg =
4109 getArgDescriptor(B, AMDGPUFunctionArgInfo::QUEUE_PTR);
4112 MachineRegisterInfo &MRI = *B.getMRI();
4113 Register SGPR01(AMDGPU::SGPR0_SGPR1);
4114 Register LiveIn = getLiveInRegister(
4115 B, MRI, SGPR01, LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64),
4116 /*InsertLiveInCopy=*/false);
4117 if (!loadInputValue(LiveIn, B, Arg))
4119 B.buildCopy(SGPR01, LiveIn);
4120 B.buildInstr(AMDGPU::S_TRAP)
4121 .addImm(GCNSubtarget::TrapIDLLVMTrap)
4122 .addReg(SGPR01, RegState::Implicit);
4125 MI.eraseFromParent();
4129 bool AMDGPULegalizerInfo::legalizeDebugTrapIntrinsic(
4130 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
4131 // Is non-HSA path or trap-handler disabled? then, report a warning
4133 if (ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbiHsa ||
4134 !ST.isTrapHandlerEnabled()) {
4135 DiagnosticInfoUnsupported NoTrap(B.getMF().getFunction(),
4136 "debugtrap handler not supported",
4137 MI.getDebugLoc(), DS_Warning);
4138 LLVMContext &Ctx = B.getMF().getFunction().getContext();
4139 Ctx.diagnose(NoTrap);
4141 // Insert debug-trap instruction
4142 B.buildInstr(AMDGPU::S_TRAP).addImm(GCNSubtarget::TrapIDLLVMDebugTrap);
4145 MI.eraseFromParent();
4149 bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
4150 MachineInstr &MI) const {
4151 MachineIRBuilder &B = Helper.MIRBuilder;
4152 MachineRegisterInfo &MRI = *B.getMRI();
4154 // Replace the use G_BRCOND with the exec manipulate and branch pseudos.
4155 auto IntrID = MI.getIntrinsicID();
4157 case Intrinsic::amdgcn_if:
4158 case Intrinsic::amdgcn_else: {
4159 MachineInstr *Br = nullptr;
4160 MachineBasicBlock *UncondBrTarget = nullptr;
4161 if (MachineInstr *BrCond = verifyCFIntrinsic(MI, MRI, Br, UncondBrTarget)) {
4162 const SIRegisterInfo *TRI
4163 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
4165 Register Def = MI.getOperand(1).getReg();
4166 Register Use = MI.getOperand(3).getReg();
4168 MachineBasicBlock *CondBrTarget = BrCond->getOperand(1).getMBB();
4169 B.setInsertPt(B.getMBB(), BrCond->getIterator());
4170 if (IntrID == Intrinsic::amdgcn_if) {
4171 B.buildInstr(AMDGPU::SI_IF)
4174 .addMBB(UncondBrTarget);
4176 B.buildInstr(AMDGPU::SI_ELSE)
4179 .addMBB(UncondBrTarget)
4184 Br->getOperand(0).setMBB(CondBrTarget);
4186 // The IRTranslator skips inserting the G_BR for fallthrough cases, but
4187 // since we're swapping branch targets it needs to be reinserted.
4188 // FIXME: IRTranslator should probably not do this
4189 B.buildBr(*CondBrTarget);
4192 MRI.setRegClass(Def, TRI->getWaveMaskRegClass());
4193 MRI.setRegClass(Use, TRI->getWaveMaskRegClass());
4194 MI.eraseFromParent();
4195 BrCond->eraseFromParent();
4201 case Intrinsic::amdgcn_loop: {
4202 MachineInstr *Br = nullptr;
4203 MachineBasicBlock *UncondBrTarget = nullptr;
4204 if (MachineInstr *BrCond = verifyCFIntrinsic(MI, MRI, Br, UncondBrTarget)) {
4205 const SIRegisterInfo *TRI
4206 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
4208 MachineBasicBlock *CondBrTarget = BrCond->getOperand(1).getMBB();
4209 Register Reg = MI.getOperand(2).getReg();
4211 B.setInsertPt(B.getMBB(), BrCond->getIterator());
4212 B.buildInstr(AMDGPU::SI_LOOP)
4214 .addMBB(UncondBrTarget);
4217 Br->getOperand(0).setMBB(CondBrTarget);
4219 B.buildBr(*CondBrTarget);
4221 MI.eraseFromParent();
4222 BrCond->eraseFromParent();
4223 MRI.setRegClass(Reg, TRI->getWaveMaskRegClass());
4229 case Intrinsic::amdgcn_kernarg_segment_ptr:
4230 if (!AMDGPU::isKernel(B.getMF().getFunction().getCallingConv())) {
4231 // This only makes sense to call in a kernel, so just lower to null.
4232 B.buildConstant(MI.getOperand(0).getReg(), 0);
4233 MI.eraseFromParent();
4237 return legalizePreloadedArgIntrin(
4238 MI, MRI, B, AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
4239 case Intrinsic::amdgcn_implicitarg_ptr:
4240 return legalizeImplicitArgPtr(MI, MRI, B);
4241 case Intrinsic::amdgcn_workitem_id_x:
4242 return legalizePreloadedArgIntrin(MI, MRI, B,
4243 AMDGPUFunctionArgInfo::WORKITEM_ID_X);
4244 case Intrinsic::amdgcn_workitem_id_y:
4245 return legalizePreloadedArgIntrin(MI, MRI, B,
4246 AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
4247 case Intrinsic::amdgcn_workitem_id_z:
4248 return legalizePreloadedArgIntrin(MI, MRI, B,
4249 AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
4250 case Intrinsic::amdgcn_workgroup_id_x:
4251 return legalizePreloadedArgIntrin(MI, MRI, B,
4252 AMDGPUFunctionArgInfo::WORKGROUP_ID_X);
4253 case Intrinsic::amdgcn_workgroup_id_y:
4254 return legalizePreloadedArgIntrin(MI, MRI, B,
4255 AMDGPUFunctionArgInfo::WORKGROUP_ID_Y);
4256 case Intrinsic::amdgcn_workgroup_id_z:
4257 return legalizePreloadedArgIntrin(MI, MRI, B,
4258 AMDGPUFunctionArgInfo::WORKGROUP_ID_Z);
4259 case Intrinsic::amdgcn_dispatch_ptr:
4260 return legalizePreloadedArgIntrin(MI, MRI, B,
4261 AMDGPUFunctionArgInfo::DISPATCH_PTR);
4262 case Intrinsic::amdgcn_queue_ptr:
4263 return legalizePreloadedArgIntrin(MI, MRI, B,
4264 AMDGPUFunctionArgInfo::QUEUE_PTR);
4265 case Intrinsic::amdgcn_implicit_buffer_ptr:
4266 return legalizePreloadedArgIntrin(
4267 MI, MRI, B, AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR);
4268 case Intrinsic::amdgcn_dispatch_id:
4269 return legalizePreloadedArgIntrin(MI, MRI, B,
4270 AMDGPUFunctionArgInfo::DISPATCH_ID);
4271 case Intrinsic::amdgcn_fdiv_fast:
4272 return legalizeFDIVFastIntrin(MI, MRI, B);
4273 case Intrinsic::amdgcn_is_shared:
4274 return legalizeIsAddrSpace(MI, MRI, B, AMDGPUAS::LOCAL_ADDRESS);
4275 case Intrinsic::amdgcn_is_private:
4276 return legalizeIsAddrSpace(MI, MRI, B, AMDGPUAS::PRIVATE_ADDRESS);
4277 case Intrinsic::amdgcn_wavefrontsize: {
4278 B.buildConstant(MI.getOperand(0), ST.getWavefrontSize());
4279 MI.eraseFromParent();
4282 case Intrinsic::amdgcn_s_buffer_load:
4283 return legalizeSBufferLoad(MI, B, Helper.Observer);
4284 case Intrinsic::amdgcn_raw_buffer_store:
4285 case Intrinsic::amdgcn_struct_buffer_store:
4286 return legalizeBufferStore(MI, MRI, B, false, false);
4287 case Intrinsic::amdgcn_raw_buffer_store_format:
4288 case Intrinsic::amdgcn_struct_buffer_store_format:
4289 return legalizeBufferStore(MI, MRI, B, false, true);
4290 case Intrinsic::amdgcn_raw_tbuffer_store:
4291 case Intrinsic::amdgcn_struct_tbuffer_store:
4292 return legalizeBufferStore(MI, MRI, B, true, true);
4293 case Intrinsic::amdgcn_raw_buffer_load:
4294 case Intrinsic::amdgcn_struct_buffer_load:
4295 return legalizeBufferLoad(MI, MRI, B, false, false);
4296 case Intrinsic::amdgcn_raw_buffer_load_format:
4297 case Intrinsic::amdgcn_struct_buffer_load_format:
4298 return legalizeBufferLoad(MI, MRI, B, true, false);
4299 case Intrinsic::amdgcn_raw_tbuffer_load:
4300 case Intrinsic::amdgcn_struct_tbuffer_load:
4301 return legalizeBufferLoad(MI, MRI, B, true, true);
4302 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
4303 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
4304 case Intrinsic::amdgcn_raw_buffer_atomic_add:
4305 case Intrinsic::amdgcn_struct_buffer_atomic_add:
4306 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
4307 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
4308 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
4309 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
4310 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
4311 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
4312 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
4313 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
4314 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
4315 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
4316 case Intrinsic::amdgcn_raw_buffer_atomic_and:
4317 case Intrinsic::amdgcn_struct_buffer_atomic_and:
4318 case Intrinsic::amdgcn_raw_buffer_atomic_or:
4319 case Intrinsic::amdgcn_struct_buffer_atomic_or:
4320 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
4321 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
4322 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
4323 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
4324 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
4325 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
4326 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
4327 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
4328 return legalizeBufferAtomic(MI, B, IntrID);
4329 case Intrinsic::amdgcn_atomic_inc:
4330 return legalizeAtomicIncDec(MI, B, true);
4331 case Intrinsic::amdgcn_atomic_dec:
4332 return legalizeAtomicIncDec(MI, B, false);
4333 case Intrinsic::trap:
4334 return legalizeTrapIntrinsic(MI, MRI, B);
4335 case Intrinsic::debugtrap:
4336 return legalizeDebugTrapIntrinsic(MI, MRI, B);
4338 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
4339 AMDGPU::getImageDimIntrinsicInfo(IntrID))
4340 return legalizeImageIntrinsic(MI, B, Helper.Observer, ImageDimIntr);