1 //===-- SIModeRegister.cpp - Mode Register --------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 /// This pass inserts changes to the Mode register settings as required.
10 /// Note that currently it only deals with the Double Precision Floating Point
11 /// rounding mode setting, but is intended to be generic enough to be easily
14 //===----------------------------------------------------------------------===//
17 #include "GCNSubtarget.h"
18 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/CodeGen/MachineFunctionPass.h"
23 #define DEBUG_TYPE "si-mode-register"
25 STATISTIC(NumSetregInserted, "Number of setreg of mode register inserted.");
30 // Mask is a bitmask where a '1' indicates the corresponding Mode bit has a
35 Status() : Mask(0), Mode(0){};
37 Status(unsigned NewMask, unsigned NewMode) : Mask(NewMask), Mode(NewMode) {
41 // merge two status values such that only values that don't conflict are
43 Status merge(const Status &S) const {
44 return Status((Mask | S.Mask), ((Mode & ~S.Mask) | (S.Mode & S.Mask)));
47 // merge an unknown value by using the unknown value's mask to remove bits
49 Status mergeUnknown(unsigned newMask) {
50 return Status(Mask & ~newMask, Mode & ~newMask);
53 // intersect two Status values to produce a mode and mask that is a subset
55 Status intersect(const Status &S) const {
56 unsigned NewMask = (Mask & S.Mask) & (Mode ^ ~S.Mode);
57 unsigned NewMode = (Mode & NewMask);
58 return Status(NewMask, NewMode);
61 // produce the delta required to change the Mode to the required Mode
62 Status delta(const Status &S) const {
63 return Status((S.Mask & (Mode ^ S.Mode)) | (~Mask & S.Mask), S.Mode);
66 bool operator==(const Status &S) const {
67 return (Mask == S.Mask) && (Mode == S.Mode);
70 bool operator!=(const Status &S) const { return !(*this == S); }
72 bool isCompatible(Status &S) {
73 return ((Mask & S.Mask) == S.Mask) && ((Mode & S.Mask) == S.Mode);
76 bool isCombinable(Status &S) { return !(Mask & S.Mask) || isCompatible(S); }
81 // The Status that represents the mode register settings required by the
82 // FirstInsertionPoint (if any) in this block. Calculated in Phase 1.
85 // The Status that represents the net changes to the Mode register made by
86 // this block, Calculated in Phase 1.
89 // The Status that represents the mode register settings on exit from this
90 // block. Calculated in Phase 2.
93 // The Status that represents the intersection of exit Mode register settings
94 // from all predecessor blocks. Calculated in Phase 2, and used by Phase 3.
97 // In Phase 1 we record the first instruction that has a mode requirement,
98 // which is used in Phase 3 if we need to insert a mode change.
99 MachineInstr *FirstInsertionPoint;
101 // A flag to indicate whether an Exit value has been set (we can't tell by
102 // examining the Exit value itself as all values may be valid results).
105 BlockData() : FirstInsertionPoint(nullptr), ExitSet(false){};
110 class SIModeRegister : public MachineFunctionPass {
114 std::vector<std::unique_ptr<BlockData>> BlockInfo;
115 std::queue<MachineBasicBlock *> Phase2List;
117 // The default mode register setting currently only caters for the floating
118 // point double precision rounding mode.
119 // We currently assume the default rounding mode is Round to Nearest
120 // NOTE: this should come from a per function rounding mode setting once such
122 unsigned DefaultMode = FP_ROUND_ROUND_TO_NEAREST;
123 Status DefaultStatus =
124 Status(FP_ROUND_MODE_DP(0x3), FP_ROUND_MODE_DP(DefaultMode));
126 bool Changed = false;
129 SIModeRegister() : MachineFunctionPass(ID) {}
131 bool runOnMachineFunction(MachineFunction &MF) override;
133 void getAnalysisUsage(AnalysisUsage &AU) const override {
134 AU.setPreservesCFG();
135 MachineFunctionPass::getAnalysisUsage(AU);
138 void processBlockPhase1(MachineBasicBlock &MBB, const SIInstrInfo *TII);
140 void processBlockPhase2(MachineBasicBlock &MBB, const SIInstrInfo *TII);
142 void processBlockPhase3(MachineBasicBlock &MBB, const SIInstrInfo *TII);
144 Status getInstructionMode(MachineInstr &MI, const SIInstrInfo *TII);
146 void insertSetreg(MachineBasicBlock &MBB, MachineInstr *I,
147 const SIInstrInfo *TII, Status InstrMode);
149 } // End anonymous namespace.
151 INITIALIZE_PASS(SIModeRegister, DEBUG_TYPE,
152 "Insert required mode register values", false, false)
154 char SIModeRegister::ID = 0;
156 char &llvm::SIModeRegisterID = SIModeRegister::ID;
158 FunctionPass *llvm::createSIModeRegisterPass() { return new SIModeRegister(); }
160 // Determine the Mode register setting required for this instruction.
161 // Instructions which don't use the Mode register return a null Status.
162 // Note this currently only deals with instructions that use the floating point
163 // double precision setting.
164 Status SIModeRegister::getInstructionMode(MachineInstr &MI,
165 const SIInstrInfo *TII) {
166 if (TII->usesFPDPRounding(MI) ||
167 MI.getOpcode() == AMDGPU::FPTRUNC_UPWARD_PSEUDO ||
168 MI.getOpcode() == AMDGPU::FPTRUNC_DOWNWARD_PSEUDO) {
169 switch (MI.getOpcode()) {
170 case AMDGPU::V_INTERP_P1LL_F16:
171 case AMDGPU::V_INTERP_P1LV_F16:
172 case AMDGPU::V_INTERP_P2_F16:
173 // f16 interpolation instructions need double precision round to zero
174 return Status(FP_ROUND_MODE_DP(3),
175 FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_ZERO));
176 case AMDGPU::FPTRUNC_UPWARD_PSEUDO: {
177 // Replacing the pseudo by a real instruction in place
178 if (TII->getSubtarget().hasTrue16BitInsts()) {
179 MachineBasicBlock &MBB = *MI.getParent();
180 MachineInstrBuilder B(*MBB.getParent(), MI);
181 MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_t16_e64));
182 MachineOperand Src0 = MI.getOperand(1);
184 B.addImm(0); // src0_modifiers
185 B.add(Src0); // re-add src0 operand
186 B.addImm(0); // clamp
189 MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_e32));
190 return Status(FP_ROUND_MODE_DP(3),
191 FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_INF));
193 case AMDGPU::FPTRUNC_DOWNWARD_PSEUDO: {
194 // Replacing the pseudo by a real instruction in place
195 if (TII->getSubtarget().hasTrue16BitInsts()) {
196 MachineBasicBlock &MBB = *MI.getParent();
197 MachineInstrBuilder B(*MBB.getParent(), MI);
198 MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_t16_e64));
199 MachineOperand Src0 = MI.getOperand(1);
201 B.addImm(0); // src0_modifiers
202 B.add(Src0); // re-add src0 operand
203 B.addImm(0); // clamp
206 MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_e32));
207 return Status(FP_ROUND_MODE_DP(3),
208 FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEGINF));
211 return DefaultStatus;
217 // Insert a setreg instruction to update the Mode register.
218 // It is possible (though unlikely) for an instruction to require a change to
219 // the value of disjoint parts of the Mode register when we don't know the
220 // value of the intervening bits. In that case we need to use more than one
221 // setreg instruction.
222 void SIModeRegister::insertSetreg(MachineBasicBlock &MBB, MachineInstr *MI,
223 const SIInstrInfo *TII, Status InstrMode) {
224 while (InstrMode.Mask) {
225 unsigned Offset = countTrailingZeros<unsigned>(InstrMode.Mask);
226 unsigned Width = countTrailingOnes<unsigned>(InstrMode.Mask >> Offset);
227 unsigned Value = (InstrMode.Mode >> Offset) & ((1 << Width) - 1);
228 BuildMI(MBB, MI, nullptr, TII->get(AMDGPU::S_SETREG_IMM32_B32))
230 .addImm(((Width - 1) << AMDGPU::Hwreg::WIDTH_M1_SHIFT_) |
231 (Offset << AMDGPU::Hwreg::OFFSET_SHIFT_) |
232 (AMDGPU::Hwreg::ID_MODE << AMDGPU::Hwreg::ID_SHIFT_));
235 InstrMode.Mask &= ~(((1 << Width) - 1) << Offset);
239 // In Phase 1 we iterate through the instructions of the block and for each
240 // instruction we get its mode usage. If the instruction uses the Mode register
242 // - update the Change status, which tracks the changes to the Mode register
243 // made by this block
244 // - if this instruction's requirements are compatible with the current setting
245 // of the Mode register we merge the modes
246 // - if it isn't compatible and an InsertionPoint isn't set, then we set the
247 // InsertionPoint to the current instruction, and we remember the current
249 // - if it isn't compatible and InsertionPoint is set we insert a seteg before
250 // that instruction (unless this instruction forms part of the block's
251 // entry requirements in which case the insertion is deferred until Phase 3
252 // when predecessor exit values are known), and move the insertion point to
254 // - if this is a setreg instruction we treat it as an incompatible instruction.
255 // This is sub-optimal but avoids some nasty corner cases, and is expected to
256 // occur very rarely.
257 // - on exit we have set the Require, Change, and initial Exit modes.
258 void SIModeRegister::processBlockPhase1(MachineBasicBlock &MBB,
259 const SIInstrInfo *TII) {
260 auto NewInfo = std::make_unique<BlockData>();
261 MachineInstr *InsertionPoint = nullptr;
262 // RequirePending is used to indicate whether we are collecting the initial
263 // requirements for the block, and need to defer the first InsertionPoint to
264 // Phase 3. It is set to false once we have set FirstInsertionPoint, or when
265 // we discover an explicit setreg that means this block doesn't have any
266 // initial requirements.
267 bool RequirePending = true;
269 for (MachineInstr &MI : MBB) {
270 Status InstrMode = getInstructionMode(MI, TII);
271 if (MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
272 MI.getOpcode() == AMDGPU::S_SETREG_B32_mode ||
273 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
274 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_mode) {
275 // We preserve any explicit mode register setreg instruction we encounter,
276 // as we assume it has been inserted by a higher authority (this is
277 // likely to be a very rare occurrence).
278 unsigned Dst = TII->getNamedOperand(MI, AMDGPU::OpName::simm16)->getImm();
279 if (((Dst & AMDGPU::Hwreg::ID_MASK_) >> AMDGPU::Hwreg::ID_SHIFT_) !=
280 AMDGPU::Hwreg::ID_MODE)
283 unsigned Width = ((Dst & AMDGPU::Hwreg::WIDTH_M1_MASK_) >>
284 AMDGPU::Hwreg::WIDTH_M1_SHIFT_) +
287 (Dst & AMDGPU::Hwreg::OFFSET_MASK_) >> AMDGPU::Hwreg::OFFSET_SHIFT_;
288 unsigned Mask = ((1 << Width) - 1) << Offset;
290 // If an InsertionPoint is set we will insert a setreg there.
291 if (InsertionPoint) {
292 insertSetreg(MBB, InsertionPoint, TII, IPChange.delta(NewInfo->Change));
293 InsertionPoint = nullptr;
295 // If this is an immediate then we know the value being set, but if it is
296 // not an immediate then we treat the modified bits of the mode register
298 if (MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
299 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_mode) {
300 unsigned Val = TII->getNamedOperand(MI, AMDGPU::OpName::imm)->getImm();
301 unsigned Mode = (Val << Offset) & Mask;
302 Status Setreg = Status(Mask, Mode);
303 // If we haven't already set the initial requirements for the block we
304 // don't need to as the requirements start from this explicit setreg.
305 RequirePending = false;
306 NewInfo->Change = NewInfo->Change.merge(Setreg);
308 NewInfo->Change = NewInfo->Change.mergeUnknown(Mask);
310 } else if (!NewInfo->Change.isCompatible(InstrMode)) {
311 // This instruction uses the Mode register and its requirements aren't
312 // compatible with the current mode.
313 if (InsertionPoint) {
314 // If the required mode change cannot be included in the current
315 // InsertionPoint changes, we need a setreg and start a new
317 if (!IPChange.delta(NewInfo->Change).isCombinable(InstrMode)) {
318 if (RequirePending) {
319 // This is the first insertionPoint in the block so we will defer
320 // the insertion of the setreg to Phase 3 where we know whether or
321 // not it is actually needed.
322 NewInfo->FirstInsertionPoint = InsertionPoint;
323 NewInfo->Require = NewInfo->Change;
324 RequirePending = false;
326 insertSetreg(MBB, InsertionPoint, TII,
327 IPChange.delta(NewInfo->Change));
328 IPChange = NewInfo->Change;
330 // Set the new InsertionPoint
331 InsertionPoint = &MI;
333 NewInfo->Change = NewInfo->Change.merge(InstrMode);
335 // No InsertionPoint is currently set - this is either the first in
336 // the block or we have previously seen an explicit setreg.
337 InsertionPoint = &MI;
338 IPChange = NewInfo->Change;
339 NewInfo->Change = NewInfo->Change.merge(InstrMode);
343 if (RequirePending) {
344 // If we haven't yet set the initial requirements for the block we set them
346 NewInfo->FirstInsertionPoint = InsertionPoint;
347 NewInfo->Require = NewInfo->Change;
348 } else if (InsertionPoint) {
349 // We need to insert a setreg at the InsertionPoint
350 insertSetreg(MBB, InsertionPoint, TII, IPChange.delta(NewInfo->Change));
352 NewInfo->Exit = NewInfo->Change;
353 BlockInfo[MBB.getNumber()] = std::move(NewInfo);
356 // In Phase 2 we revisit each block and calculate the common Mode register
357 // value provided by all predecessor blocks. If the Exit value for the block
358 // is changed, then we add the successor blocks to the worklist so that the
359 // exit value is propagated.
360 void SIModeRegister::processBlockPhase2(MachineBasicBlock &MBB,
361 const SIInstrInfo *TII) {
362 bool RevisitRequired = false;
363 bool ExitSet = false;
364 unsigned ThisBlock = MBB.getNumber();
365 if (MBB.pred_empty()) {
366 // There are no predecessors, so use the default starting status.
367 BlockInfo[ThisBlock]->Pred = DefaultStatus;
370 // Build a status that is common to all the predecessors by intersecting
371 // all the predecessor exit status values.
372 // Mask bits (which represent the Mode bits with a known value) can only be
373 // added by explicit SETREG instructions or the initial default value -
374 // the intersection process may remove Mask bits.
375 // If we find a predecessor that has not yet had an exit value determined
376 // (this can happen for example if a block is its own predecessor) we defer
377 // use of that value as the Mask will be all zero, and we will revisit this
378 // block again later (unless the only predecessor without an exit value is
380 MachineBasicBlock::pred_iterator P = MBB.pred_begin(), E = MBB.pred_end();
381 MachineBasicBlock &PB = *(*P);
382 unsigned PredBlock = PB.getNumber();
383 if ((ThisBlock == PredBlock) && (std::next(P) == E)) {
384 BlockInfo[ThisBlock]->Pred = DefaultStatus;
386 } else if (BlockInfo[PredBlock]->ExitSet) {
387 BlockInfo[ThisBlock]->Pred = BlockInfo[PredBlock]->Exit;
389 } else if (PredBlock != ThisBlock)
390 RevisitRequired = true;
392 for (P = std::next(P); P != E; P = std::next(P)) {
393 MachineBasicBlock *Pred = *P;
394 unsigned PredBlock = Pred->getNumber();
395 if (BlockInfo[PredBlock]->ExitSet) {
396 if (BlockInfo[ThisBlock]->ExitSet) {
397 BlockInfo[ThisBlock]->Pred =
398 BlockInfo[ThisBlock]->Pred.intersect(BlockInfo[PredBlock]->Exit);
400 BlockInfo[ThisBlock]->Pred = BlockInfo[PredBlock]->Exit;
403 } else if (PredBlock != ThisBlock)
404 RevisitRequired = true;
408 BlockInfo[ThisBlock]->Pred.merge(BlockInfo[ThisBlock]->Change);
409 if (BlockInfo[ThisBlock]->Exit != TmpStatus) {
410 BlockInfo[ThisBlock]->Exit = TmpStatus;
411 // Add the successors to the work list so we can propagate the changed exit
413 for (MachineBasicBlock *Succ : MBB.successors())
414 Phase2List.push(Succ);
416 BlockInfo[ThisBlock]->ExitSet = ExitSet;
418 Phase2List.push(&MBB);
421 // In Phase 3 we revisit each block and if it has an insertion point defined we
422 // check whether the predecessor mode meets the block's entry requirements. If
423 // not we insert an appropriate setreg instruction to modify the Mode register.
424 void SIModeRegister::processBlockPhase3(MachineBasicBlock &MBB,
425 const SIInstrInfo *TII) {
426 unsigned ThisBlock = MBB.getNumber();
427 if (!BlockInfo[ThisBlock]->Pred.isCompatible(BlockInfo[ThisBlock]->Require)) {
429 BlockInfo[ThisBlock]->Pred.delta(BlockInfo[ThisBlock]->Require);
430 if (BlockInfo[ThisBlock]->FirstInsertionPoint)
431 insertSetreg(MBB, BlockInfo[ThisBlock]->FirstInsertionPoint, TII, Delta);
433 insertSetreg(MBB, &MBB.instr_front(), TII, Delta);
437 bool SIModeRegister::runOnMachineFunction(MachineFunction &MF) {
438 BlockInfo.resize(MF.getNumBlockIDs());
439 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
440 const SIInstrInfo *TII = ST.getInstrInfo();
442 // Processing is performed in a number of phases
444 // Phase 1 - determine the initial mode required by each block, and add setreg
445 // instructions for intra block requirements.
446 for (MachineBasicBlock &BB : MF)
447 processBlockPhase1(BB, TII);
449 // Phase 2 - determine the exit mode from each block. We add all blocks to the
450 // list here, but will also add any that need to be revisited during Phase 2
452 for (MachineBasicBlock &BB : MF)
453 Phase2List.push(&BB);
454 while (!Phase2List.empty()) {
455 processBlockPhase2(*Phase2List.front(), TII);
459 // Phase 3 - add an initial setreg to each block where the required entry mode
460 // is not satisfied by the exit mode of all its predecessors.
461 for (MachineBasicBlock &BB : MF)
462 processBlockPhase3(BB, TII);