1 //===-- SIWholeQuadMode.cpp - enter and suspend whole quad mode -----------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// This pass adds instructions to enable whole quad mode for pixel
11 /// shaders, and whole wavefront mode for all programs.
13 /// Whole quad mode is required for derivative computations, but it interferes
14 /// with shader side effects (stores and atomics). This pass is run on the
15 /// scheduled machine IR but before register coalescing, so that machine SSA is
16 /// available for analysis. It ensures that WQM is enabled when necessary, but
17 /// disabled around stores and atomics.
19 /// When necessary, this pass creates a function prolog
21 /// S_MOV_B64 LiveMask, EXEC
22 /// S_WQM_B64 EXEC, EXEC
24 /// to enter WQM at the top of the function and surrounds blocks of Exact
27 /// S_AND_SAVEEXEC_B64 Tmp, LiveMask
29 /// S_MOV_B64 EXEC, Tmp
31 /// We also compute when a sequence of instructions requires Whole Wavefront
32 /// Mode (WWM) and insert instructions to save and restore it:
34 /// S_OR_SAVEEXEC_B64 Tmp, -1
36 /// S_MOV_B64 EXEC, Tmp
38 /// In order to avoid excessive switching during sequences of Exact
39 /// instructions, the pass first analyzes which instructions must be run in WQM
40 /// (aka which instructions produce values that lead to derivative
43 /// Basic blocks are always exited in WQM as long as some successor needs WQM.
45 /// There is room for improvement given better control flow analysis:
47 /// (1) at the top level (outside of control flow statements, and as long as
48 /// kill hasn't been used), one SGPR can be saved by recovering WQM from
49 /// the LiveMask (this is implemented for the entry block).
51 /// (2) when entire regions (e.g. if-else blocks or entire loops) only
52 /// consist of exact and don't-care instructions, the switch only has to
53 /// be done at the entry and exit points rather than potentially in each
54 /// block of the region.
56 //===----------------------------------------------------------------------===//
59 #include "AMDGPUSubtarget.h"
60 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
61 #include "SIInstrInfo.h"
62 #include "SIMachineFunctionInfo.h"
63 #include "llvm/ADT/DenseMap.h"
64 #include "llvm/ADT/PostOrderIterator.h"
65 #include "llvm/ADT/SmallVector.h"
66 #include "llvm/ADT/StringRef.h"
67 #include "llvm/CodeGen/LiveInterval.h"
68 #include "llvm/CodeGen/LiveIntervals.h"
69 #include "llvm/CodeGen/MachineBasicBlock.h"
70 #include "llvm/CodeGen/MachineFunction.h"
71 #include "llvm/CodeGen/MachineFunctionPass.h"
72 #include "llvm/CodeGen/MachineInstr.h"
73 #include "llvm/CodeGen/MachineInstrBuilder.h"
74 #include "llvm/CodeGen/MachineOperand.h"
75 #include "llvm/CodeGen/MachineRegisterInfo.h"
76 #include "llvm/CodeGen/SlotIndexes.h"
77 #include "llvm/CodeGen/TargetRegisterInfo.h"
78 #include "llvm/IR/CallingConv.h"
79 #include "llvm/IR/DebugLoc.h"
80 #include "llvm/InitializePasses.h"
81 #include "llvm/MC/MCRegisterInfo.h"
82 #include "llvm/Pass.h"
83 #include "llvm/Support/Debug.h"
84 #include "llvm/Support/raw_ostream.h"
90 #define DEBUG_TYPE "si-wqm"
104 explicit PrintState(int State) : State(State) {}
108 static raw_ostream &operator<<(raw_ostream &OS, const PrintState &PS) {
109 if (PS.State & StateWQM)
111 if (PS.State & StateWWM) {
112 if (PS.State & StateWQM)
116 if (PS.State & StateExact) {
117 if (PS.State & (StateWQM | StateWWM))
139 MachineBasicBlock *MBB = nullptr;
140 MachineInstr *MI = nullptr;
142 WorkItem() = default;
143 WorkItem(MachineBasicBlock *MBB) : MBB(MBB) {}
144 WorkItem(MachineInstr *MI) : MI(MI) {}
147 class SIWholeQuadMode : public MachineFunctionPass {
149 CallingConv::ID CallingConv;
150 const SIInstrInfo *TII;
151 const SIRegisterInfo *TRI;
152 const GCNSubtarget *ST;
153 MachineRegisterInfo *MRI;
156 DenseMap<const MachineInstr *, InstrInfo> Instructions;
157 DenseMap<MachineBasicBlock *, BlockInfo> Blocks;
158 SmallVector<MachineInstr *, 1> LiveMaskQueries;
159 SmallVector<MachineInstr *, 4> LowerToMovInstrs;
160 SmallVector<MachineInstr *, 4> LowerToCopyInstrs;
164 void markInstruction(MachineInstr &MI, char Flag,
165 std::vector<WorkItem> &Worklist);
166 void markInstructionUses(const MachineInstr &MI, char Flag,
167 std::vector<WorkItem> &Worklist);
168 char scanInstructions(MachineFunction &MF, std::vector<WorkItem> &Worklist);
169 void propagateInstruction(MachineInstr &MI, std::vector<WorkItem> &Worklist);
170 void propagateBlock(MachineBasicBlock &MBB, std::vector<WorkItem> &Worklist);
171 char analyzeFunction(MachineFunction &MF);
173 bool requiresCorrectState(const MachineInstr &MI) const;
175 MachineBasicBlock::iterator saveSCC(MachineBasicBlock &MBB,
176 MachineBasicBlock::iterator Before);
177 MachineBasicBlock::iterator
178 prepareInsertion(MachineBasicBlock &MBB, MachineBasicBlock::iterator First,
179 MachineBasicBlock::iterator Last, bool PreferLast,
181 void toExact(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
182 unsigned SaveWQM, unsigned LiveMaskReg);
183 void toWQM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
185 void toWWM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
187 void fromWWM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
189 void processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, bool isEntry);
191 void lowerLiveMaskQueries(unsigned LiveMaskReg);
192 void lowerCopyInstrs();
198 MachineFunctionPass(ID) { }
200 bool runOnMachineFunction(MachineFunction &MF) override;
202 StringRef getPassName() const override { return "SI Whole Quad Mode"; }
204 void getAnalysisUsage(AnalysisUsage &AU) const override {
205 AU.addRequired<LiveIntervals>();
206 AU.addPreserved<SlotIndexes>();
207 AU.addPreserved<LiveIntervals>();
208 AU.setPreservesCFG();
209 MachineFunctionPass::getAnalysisUsage(AU);
213 } // end anonymous namespace
215 char SIWholeQuadMode::ID = 0;
217 INITIALIZE_PASS_BEGIN(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false,
219 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
220 INITIALIZE_PASS_END(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false,
223 char &llvm::SIWholeQuadModeID = SIWholeQuadMode::ID;
225 FunctionPass *llvm::createSIWholeQuadModePass() {
226 return new SIWholeQuadMode;
230 LLVM_DUMP_METHOD void SIWholeQuadMode::printInfo() {
231 for (const auto &BII : Blocks) {
233 << printMBBReference(*BII.first) << ":\n"
234 << " InNeeds = " << PrintState(BII.second.InNeeds)
235 << ", Needs = " << PrintState(BII.second.Needs)
236 << ", OutNeeds = " << PrintState(BII.second.OutNeeds) << "\n\n";
238 for (const MachineInstr &MI : *BII.first) {
239 auto III = Instructions.find(&MI);
240 if (III == Instructions.end())
243 dbgs() << " " << MI << " Needs = " << PrintState(III->second.Needs)
244 << ", OutNeeds = " << PrintState(III->second.OutNeeds) << '\n';
250 void SIWholeQuadMode::markInstruction(MachineInstr &MI, char Flag,
251 std::vector<WorkItem> &Worklist) {
252 InstrInfo &II = Instructions[&MI];
254 assert(!(Flag & StateExact) && Flag != 0);
256 // Remove any disabled states from the flag. The user that required it gets
257 // an undefined value in the helper lanes. For example, this can happen if
258 // the result of an atomic is used by instruction that requires WQM, where
259 // ignoring the request for WQM is correct as per the relevant specs.
260 Flag &= ~II.Disabled;
262 // Ignore if the flag is already encompassed by the existing needs, or we
263 // just disabled everything.
264 if ((II.Needs & Flag) == Flag)
268 Worklist.push_back(&MI);
271 /// Mark all instructions defining the uses in \p MI with \p Flag.
272 void SIWholeQuadMode::markInstructionUses(const MachineInstr &MI, char Flag,
273 std::vector<WorkItem> &Worklist) {
274 for (const MachineOperand &Use : MI.uses()) {
275 if (!Use.isReg() || !Use.isUse())
278 Register Reg = Use.getReg();
280 // Handle physical registers that we need to track; this is mostly relevant
281 // for VCC, which can appear as the (implicit) input of a uniform branch,
282 // e.g. when a loop counter is stored in a VGPR.
283 if (!Register::isVirtualRegister(Reg)) {
284 if (Reg == AMDGPU::EXEC || Reg == AMDGPU::EXEC_LO)
287 for (MCRegUnitIterator RegUnit(Reg, TRI); RegUnit.isValid(); ++RegUnit) {
288 LiveRange &LR = LIS->getRegUnit(*RegUnit);
289 const VNInfo *Value = LR.Query(LIS->getInstructionIndex(MI)).valueIn();
293 // Since we're in machine SSA, we do not need to track physical
294 // registers across basic blocks.
295 if (Value->isPHIDef())
298 markInstruction(*LIS->getInstructionFromIndex(Value->def), Flag,
305 for (MachineInstr &DefMI : MRI->def_instructions(Use.getReg()))
306 markInstruction(DefMI, Flag, Worklist);
310 // Scan instructions to determine which ones require an Exact execmask and
311 // which ones seed WQM requirements.
312 char SIWholeQuadMode::scanInstructions(MachineFunction &MF,
313 std::vector<WorkItem> &Worklist) {
314 char GlobalFlags = 0;
315 bool WQMOutputs = MF.getFunction().hasFnAttribute("amdgpu-ps-wqm-outputs");
316 SmallVector<MachineInstr *, 4> SetInactiveInstrs;
317 SmallVector<MachineInstr *, 4> SoftWQMInstrs;
319 // We need to visit the basic blocks in reverse post-order so that we visit
320 // defs before uses, in particular so that we don't accidentally mark an
321 // instruction as needing e.g. WQM before visiting it and realizing it needs
323 ReversePostOrderTraversal<MachineFunction *> RPOT(&MF);
324 for (auto BI = RPOT.begin(), BE = RPOT.end(); BI != BE; ++BI) {
325 MachineBasicBlock &MBB = **BI;
326 BlockInfo &BBI = Blocks[&MBB];
328 for (auto II = MBB.begin(), IE = MBB.end(); II != IE; ++II) {
329 MachineInstr &MI = *II;
330 InstrInfo &III = Instructions[&MI];
331 unsigned Opcode = MI.getOpcode();
334 if (TII->isWQM(Opcode)) {
335 // Sampling instructions don't need to produce results for all pixels
336 // in a quad, they just require all inputs of a quad to have been
337 // computed for derivatives.
338 markInstructionUses(MI, StateWQM, Worklist);
339 GlobalFlags |= StateWQM;
341 } else if (Opcode == AMDGPU::WQM) {
342 // The WQM intrinsic requires its output to have all the helper lanes
343 // correct, so we need it to be in WQM.
345 LowerToCopyInstrs.push_back(&MI);
346 } else if (Opcode == AMDGPU::SOFT_WQM) {
347 LowerToCopyInstrs.push_back(&MI);
348 SoftWQMInstrs.push_back(&MI);
350 } else if (Opcode == AMDGPU::WWM) {
351 // The WWM intrinsic doesn't make the same guarantee, and plus it needs
352 // to be executed in WQM or Exact so that its copy doesn't clobber
354 markInstructionUses(MI, StateWWM, Worklist);
355 GlobalFlags |= StateWWM;
356 LowerToMovInstrs.push_back(&MI);
358 } else if (Opcode == AMDGPU::V_SET_INACTIVE_B32 ||
359 Opcode == AMDGPU::V_SET_INACTIVE_B64) {
360 III.Disabled = StateWWM;
361 MachineOperand &Inactive = MI.getOperand(2);
362 if (Inactive.isReg()) {
363 if (Inactive.isUndef()) {
364 LowerToCopyInstrs.push_back(&MI);
366 Register Reg = Inactive.getReg();
367 if (Register::isVirtualRegister(Reg)) {
368 for (MachineInstr &DefMI : MRI->def_instructions(Reg))
369 markInstruction(DefMI, StateWWM, Worklist);
373 SetInactiveInstrs.push_back(&MI);
375 } else if (TII->isDisableWQM(MI)) {
376 BBI.Needs |= StateExact;
377 if (!(BBI.InNeeds & StateExact)) {
378 BBI.InNeeds |= StateExact;
379 Worklist.push_back(&MBB);
381 GlobalFlags |= StateExact;
382 III.Disabled = StateWQM | StateWWM;
385 if (Opcode == AMDGPU::SI_PS_LIVE) {
386 LiveMaskQueries.push_back(&MI);
387 } else if (WQMOutputs) {
388 // The function is in machine SSA form, which means that physical
389 // VGPRs correspond to shader inputs and outputs. Inputs are
390 // only used, outputs are only defined.
391 for (const MachineOperand &MO : MI.defs()) {
395 Register Reg = MO.getReg();
397 if (!Register::isVirtualRegister(Reg) &&
398 TRI->hasVectorRegisters(TRI->getPhysRegClass(Reg))) {
409 markInstruction(MI, Flags, Worklist);
410 GlobalFlags |= Flags;
414 // Mark sure that any SET_INACTIVE instructions are computed in WQM if WQM is
415 // ever used anywhere in the function. This implements the corresponding
416 // semantics of @llvm.amdgcn.set.inactive.
417 // Similarly for SOFT_WQM instructions, implementing @llvm.amdgcn.softwqm.
418 if (GlobalFlags & StateWQM) {
419 for (MachineInstr *MI : SetInactiveInstrs)
420 markInstruction(*MI, StateWQM, Worklist);
421 for (MachineInstr *MI : SoftWQMInstrs)
422 markInstruction(*MI, StateWQM, Worklist);
428 void SIWholeQuadMode::propagateInstruction(MachineInstr &MI,
429 std::vector<WorkItem>& Worklist) {
430 MachineBasicBlock *MBB = MI.getParent();
431 InstrInfo II = Instructions[&MI]; // take a copy to prevent dangling references
432 BlockInfo &BI = Blocks[MBB];
434 // Control flow-type instructions and stores to temporary memory that are
435 // followed by WQM computations must themselves be in WQM.
436 if ((II.OutNeeds & StateWQM) && !(II.Disabled & StateWQM) &&
437 (MI.isTerminator() || (TII->usesVM_CNT(MI) && MI.mayStore()))) {
438 Instructions[&MI].Needs = StateWQM;
442 // Propagate to block level
443 if (II.Needs & StateWQM) {
444 BI.Needs |= StateWQM;
445 if (!(BI.InNeeds & StateWQM)) {
446 BI.InNeeds |= StateWQM;
447 Worklist.push_back(MBB);
451 // Propagate backwards within block
452 if (MachineInstr *PrevMI = MI.getPrevNode()) {
453 char InNeeds = (II.Needs & ~StateWWM) | II.OutNeeds;
454 if (!PrevMI->isPHI()) {
455 InstrInfo &PrevII = Instructions[PrevMI];
456 if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) {
457 PrevII.OutNeeds |= InNeeds;
458 Worklist.push_back(PrevMI);
463 // Propagate WQM flag to instruction inputs
464 assert(!(II.Needs & StateExact));
467 markInstructionUses(MI, II.Needs, Worklist);
469 // Ensure we process a block containing WWM, even if it does not require any
471 if (II.Needs & StateWWM)
472 BI.Needs |= StateWWM;
475 void SIWholeQuadMode::propagateBlock(MachineBasicBlock &MBB,
476 std::vector<WorkItem>& Worklist) {
477 BlockInfo BI = Blocks[&MBB]; // Make a copy to prevent dangling references.
479 // Propagate through instructions
481 MachineInstr *LastMI = &*MBB.rbegin();
482 InstrInfo &LastII = Instructions[LastMI];
483 if ((LastII.OutNeeds | BI.OutNeeds) != LastII.OutNeeds) {
484 LastII.OutNeeds |= BI.OutNeeds;
485 Worklist.push_back(LastMI);
489 // Predecessor blocks must provide for our WQM/Exact needs.
490 for (MachineBasicBlock *Pred : MBB.predecessors()) {
491 BlockInfo &PredBI = Blocks[Pred];
492 if ((PredBI.OutNeeds | BI.InNeeds) == PredBI.OutNeeds)
495 PredBI.OutNeeds |= BI.InNeeds;
496 PredBI.InNeeds |= BI.InNeeds;
497 Worklist.push_back(Pred);
500 // All successors must be prepared to accept the same set of WQM/Exact data.
501 for (MachineBasicBlock *Succ : MBB.successors()) {
502 BlockInfo &SuccBI = Blocks[Succ];
503 if ((SuccBI.InNeeds | BI.OutNeeds) == SuccBI.InNeeds)
506 SuccBI.InNeeds |= BI.OutNeeds;
507 Worklist.push_back(Succ);
511 char SIWholeQuadMode::analyzeFunction(MachineFunction &MF) {
512 std::vector<WorkItem> Worklist;
513 char GlobalFlags = scanInstructions(MF, Worklist);
515 while (!Worklist.empty()) {
516 WorkItem WI = Worklist.back();
520 propagateInstruction(*WI.MI, Worklist);
522 propagateBlock(*WI.MBB, Worklist);
528 /// Whether \p MI really requires the exec state computed during analysis.
530 /// Scalar instructions must occasionally be marked WQM for correct propagation
531 /// (e.g. thread masks leading up to branches), but when it comes to actual
532 /// execution, they don't care about EXEC.
533 bool SIWholeQuadMode::requiresCorrectState(const MachineInstr &MI) const {
534 if (MI.isTerminator())
537 // Skip instructions that are not affected by EXEC
538 if (TII->isScalarUnit(MI))
541 // Generic instructions such as COPY will either disappear by register
542 // coalescing or be lowered to SALU or VALU instructions.
543 if (MI.isTransient()) {
544 if (MI.getNumExplicitOperands() >= 1) {
545 const MachineOperand &Op = MI.getOperand(0);
547 if (TRI->isSGPRReg(*MRI, Op.getReg())) {
548 // SGPR instructions are not affected by EXEC
558 MachineBasicBlock::iterator
559 SIWholeQuadMode::saveSCC(MachineBasicBlock &MBB,
560 MachineBasicBlock::iterator Before) {
561 Register SaveReg = MRI->createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
564 BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), SaveReg)
565 .addReg(AMDGPU::SCC);
566 MachineInstr *Restore =
567 BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::SCC)
570 LIS->InsertMachineInstrInMaps(*Save);
571 LIS->InsertMachineInstrInMaps(*Restore);
572 LIS->createAndComputeVirtRegInterval(SaveReg);
577 // Return an iterator in the (inclusive) range [First, Last] at which
578 // instructions can be safely inserted, keeping in mind that some of the
579 // instructions we want to add necessarily clobber SCC.
580 MachineBasicBlock::iterator SIWholeQuadMode::prepareInsertion(
581 MachineBasicBlock &MBB, MachineBasicBlock::iterator First,
582 MachineBasicBlock::iterator Last, bool PreferLast, bool SaveSCC) {
584 return PreferLast ? Last : First;
586 LiveRange &LR = LIS->getRegUnit(*MCRegUnitIterator(AMDGPU::SCC, TRI));
587 auto MBBE = MBB.end();
588 SlotIndex FirstIdx = First != MBBE ? LIS->getInstructionIndex(*First)
589 : LIS->getMBBEndIdx(&MBB);
591 Last != MBBE ? LIS->getInstructionIndex(*Last) : LIS->getMBBEndIdx(&MBB);
592 SlotIndex Idx = PreferLast ? LastIdx : FirstIdx;
593 const LiveRange::Segment *S;
596 S = LR.getSegmentContaining(Idx);
601 SlotIndex Next = S->start.getBaseIndex();
606 SlotIndex Next = S->end.getNextIndex().getBaseIndex();
613 MachineBasicBlock::iterator MBBI;
615 if (MachineInstr *MI = LIS->getInstructionFromIndex(Idx))
618 assert(Idx == LIS->getMBBEndIdx(&MBB));
623 MBBI = saveSCC(MBB, MBBI);
628 void SIWholeQuadMode::toExact(MachineBasicBlock &MBB,
629 MachineBasicBlock::iterator Before,
630 unsigned SaveWQM, unsigned LiveMaskReg) {
634 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(ST->isWave32() ?
635 AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64),
637 .addReg(LiveMaskReg);
639 unsigned Exec = ST->isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
640 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(ST->isWave32() ?
641 AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64),
644 .addReg(LiveMaskReg);
647 LIS->InsertMachineInstrInMaps(*MI);
650 void SIWholeQuadMode::toWQM(MachineBasicBlock &MBB,
651 MachineBasicBlock::iterator Before,
655 unsigned Exec = ST->isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
657 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), Exec)
660 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(ST->isWave32() ?
661 AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64),
666 LIS->InsertMachineInstrInMaps(*MI);
669 void SIWholeQuadMode::toWWM(MachineBasicBlock &MBB,
670 MachineBasicBlock::iterator Before,
675 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::ENTER_WWM), SaveOrig)
677 LIS->InsertMachineInstrInMaps(*MI);
680 void SIWholeQuadMode::fromWWM(MachineBasicBlock &MBB,
681 MachineBasicBlock::iterator Before,
682 unsigned SavedOrig) {
686 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_WWM),
687 ST->isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC)
689 LIS->InsertMachineInstrInMaps(*MI);
692 void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg,
694 auto BII = Blocks.find(&MBB);
695 if (BII == Blocks.end())
698 const BlockInfo &BI = BII->second;
700 // This is a non-entry block that is WQM throughout, so no need to do
702 if (!isEntry && BI.Needs == StateWQM && BI.OutNeeds != StateExact)
705 LLVM_DEBUG(dbgs() << "\nProcessing block " << printMBBReference(MBB)
708 unsigned SavedWQMReg = 0;
709 unsigned SavedNonWWMReg = 0;
710 bool WQMFromExec = isEntry;
711 char State = (isEntry || !(BI.InNeeds & StateWQM)) ? StateExact : StateWQM;
712 char NonWWMState = 0;
713 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
715 auto II = MBB.getFirstNonPHI(), IE = MBB.end();
717 ++II; // Skip the instruction that saves LiveMask
719 // This stores the first instruction where it's safe to switch from WQM to
720 // Exact or vice versa.
721 MachineBasicBlock::iterator FirstWQM = IE;
723 // This stores the first instruction where it's safe to switch from WWM to
724 // Exact/WQM or to switch to WWM. It must always be the same as, or after,
725 // FirstWQM since if it's safe to switch to/from WWM, it must be safe to
726 // switch to/from WQM as well.
727 MachineBasicBlock::iterator FirstWWM = IE;
729 MachineBasicBlock::iterator Next = II;
730 char Needs = StateExact | StateWQM; // WWM is disabled by default
739 // First, figure out the allowed states (Needs) based on the propagated
742 MachineInstr &MI = *II;
744 if (requiresCorrectState(MI)) {
745 auto III = Instructions.find(&MI);
746 if (III != Instructions.end()) {
747 if (III->second.Needs & StateWWM)
749 else if (III->second.Needs & StateWQM)
752 Needs &= ~III->second.Disabled;
753 OutNeeds = III->second.OutNeeds;
756 // If the instruction doesn't actually need a correct EXEC, then we can
757 // safely leave WWM enabled.
758 Needs = StateExact | StateWQM | StateWWM;
761 if (MI.isTerminator() && OutNeeds == StateExact)
764 if (MI.getOpcode() == AMDGPU::SI_ELSE && BI.OutNeeds == StateExact)
765 MI.getOperand(3).setImm(1);
769 // End of basic block
770 if (BI.OutNeeds & StateWQM)
772 else if (BI.OutNeeds == StateExact)
775 Needs = StateWQM | StateExact;
778 // Now, transition if necessary.
779 if (!(Needs & State)) {
780 MachineBasicBlock::iterator First;
781 if (State == StateWWM || Needs == StateWWM) {
782 // We must switch to or from WWM
785 // We only need to switch to/from WQM, so we can use FirstWQM
789 MachineBasicBlock::iterator Before =
790 prepareInsertion(MBB, First, II, Needs == StateWQM,
791 Needs == StateExact || WQMFromExec);
793 if (State == StateWWM) {
794 assert(SavedNonWWMReg);
795 fromWWM(MBB, Before, SavedNonWWMReg);
799 if (Needs == StateWWM) {
801 SavedNonWWMReg = MRI->createVirtualRegister(BoolRC);
802 toWWM(MBB, Before, SavedNonWWMReg);
805 if (State == StateWQM && (Needs & StateExact) && !(Needs & StateWQM)) {
806 if (!WQMFromExec && (OutNeeds & StateWQM))
807 SavedWQMReg = MRI->createVirtualRegister(BoolRC);
809 toExact(MBB, Before, SavedWQMReg, LiveMaskReg);
811 } else if (State == StateExact && (Needs & StateWQM) &&
812 !(Needs & StateExact)) {
813 assert(WQMFromExec == (SavedWQMReg == 0));
815 toWQM(MBB, Before, SavedWQMReg);
818 LIS->createAndComputeVirtRegInterval(SavedWQMReg);
823 // We can get here if we transitioned from WWM to a non-WWM state that
824 // already matches our needs, but we shouldn't need to do anything.
825 assert(Needs & State);
830 if (Needs != (StateExact | StateWQM | StateWWM)) {
831 if (Needs != (StateExact | StateWQM))
842 void SIWholeQuadMode::lowerLiveMaskQueries(unsigned LiveMaskReg) {
843 for (MachineInstr *MI : LiveMaskQueries) {
844 const DebugLoc &DL = MI->getDebugLoc();
845 Register Dest = MI->getOperand(0).getReg();
847 BuildMI(*MI->getParent(), MI, DL, TII->get(AMDGPU::COPY), Dest)
848 .addReg(LiveMaskReg);
850 LIS->ReplaceMachineInstrInMaps(*MI, *Copy);
851 MI->eraseFromParent();
855 void SIWholeQuadMode::lowerCopyInstrs() {
856 for (MachineInstr *MI : LowerToMovInstrs) {
857 assert(MI->getNumExplicitOperands() == 2);
859 const Register Reg = MI->getOperand(0).getReg();
861 if (TRI->isVGPR(*MRI, Reg)) {
862 const TargetRegisterClass *regClass = Register::isVirtualRegister(Reg)
863 ? MRI->getRegClass(Reg)
864 : TRI->getPhysRegClass(Reg);
866 const unsigned MovOp = TII->getMovOpcode(regClass);
867 MI->setDesc(TII->get(MovOp));
869 // And make it implicitly depend on exec (like all VALU movs should do).
870 MI->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
872 MI->setDesc(TII->get(AMDGPU::COPY));
875 for (MachineInstr *MI : LowerToCopyInstrs) {
876 if (MI->getOpcode() == AMDGPU::V_SET_INACTIVE_B32 ||
877 MI->getOpcode() == AMDGPU::V_SET_INACTIVE_B64) {
878 assert(MI->getNumExplicitOperands() == 3);
879 // the only reason we should be here is V_SET_INACTIVE has
880 // an undef input so it is being replaced by a simple copy.
881 // There should be a second undef source that we should remove.
882 assert(MI->getOperand(2).isUndef());
883 MI->RemoveOperand(2);
884 MI->untieRegOperand(1);
886 assert(MI->getNumExplicitOperands() == 2);
889 MI->setDesc(TII->get(AMDGPU::COPY));
893 bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
894 Instructions.clear();
896 LiveMaskQueries.clear();
897 LowerToCopyInstrs.clear();
898 LowerToMovInstrs.clear();
899 CallingConv = MF.getFunction().getCallingConv();
901 ST = &MF.getSubtarget<GCNSubtarget>();
903 TII = ST->getInstrInfo();
904 TRI = &TII->getRegisterInfo();
905 MRI = &MF.getRegInfo();
906 LIS = &getAnalysis<LiveIntervals>();
908 char GlobalFlags = analyzeFunction(MF);
909 unsigned LiveMaskReg = 0;
910 unsigned Exec = ST->isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
911 if (!(GlobalFlags & StateWQM)) {
912 lowerLiveMaskQueries(Exec);
913 if (!(GlobalFlags & StateWWM) && LowerToCopyInstrs.empty() && LowerToMovInstrs.empty())
914 return !LiveMaskQueries.empty();
916 // Store a copy of the original live mask when required
917 MachineBasicBlock &Entry = MF.front();
918 MachineBasicBlock::iterator EntryMI = Entry.getFirstNonPHI();
920 if (GlobalFlags & StateExact || !LiveMaskQueries.empty()) {
921 LiveMaskReg = MRI->createVirtualRegister(TRI->getBoolRC());
922 MachineInstr *MI = BuildMI(Entry, EntryMI, DebugLoc(),
923 TII->get(AMDGPU::COPY), LiveMaskReg)
925 LIS->InsertMachineInstrInMaps(*MI);
928 lowerLiveMaskQueries(LiveMaskReg);
930 if (GlobalFlags == StateWQM) {
931 // For a shader that needs only WQM, we can just set it once.
932 BuildMI(Entry, EntryMI, DebugLoc(), TII->get(ST->isWave32() ?
933 AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64),
938 // EntryMI may become invalid here
943 LLVM_DEBUG(printInfo());
947 // Handle the general case
948 for (auto BII : Blocks)
949 processBlock(*BII.first, LiveMaskReg, BII.first == &*MF.begin());
951 // Physical registers like SCC aren't tracked by default anyway, so just
952 // removing the ranges we computed is the simplest option for maintaining
953 // the analysis results.
954 LIS->removeRegUnit(*MCRegUnitIterator(AMDGPU::SCC, TRI));