1 //===-- AArch64AdvSIMDScalar.cpp - Replace dead defs w/ zero reg --===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
9 // When profitable, replace GPR targeting i64 instructions with their
10 // AdvSIMD scalar equivalents. Generally speaking, "profitable" is defined
11 // as minimizing the number of cross-class register copies.
12 //===----------------------------------------------------------------------===//
14 //===----------------------------------------------------------------------===//
15 // TODO: Graph based predicate heuristics.
16 // Walking the instruction list linearly will get many, perhaps most, of
17 // the cases, but to do a truly thorough job of this, we need a more
18 // wholistic approach.
20 // This optimization is very similar in spirit to the register allocator's
21 // spill placement, only here we're determining where to place cross-class
22 // register copies rather than spills. As such, a similar approach is
25 // We want to build up a set of graphs of all instructions which are candidates
26 // for transformation along with instructions which generate their inputs and
27 // consume their outputs. For each edge in the graph, we assign a weight
28 // based on whether there is a copy required there (weight zero if not) and
29 // the block frequency of the block containing the defining or using
30 // instruction, whichever is less. Our optimization is then a graph problem
31 // to minimize the total weight of all the graphs, then transform instructions
32 // and add or remove copy instructions as called for to implement the
34 //===----------------------------------------------------------------------===//
37 #include "AArch64InstrInfo.h"
38 #include "AArch64RegisterInfo.h"
39 #include "AArch64Subtarget.h"
40 #include "llvm/ADT/Statistic.h"
41 #include "llvm/CodeGen/MachineFunction.h"
42 #include "llvm/CodeGen/MachineFunctionPass.h"
43 #include "llvm/CodeGen/MachineInstr.h"
44 #include "llvm/CodeGen/MachineInstrBuilder.h"
45 #include "llvm/CodeGen/MachineRegisterInfo.h"
46 #include "llvm/Support/CommandLine.h"
47 #include "llvm/Support/Debug.h"
48 #include "llvm/Support/raw_ostream.h"
51 #define DEBUG_TYPE "aarch64-simd-scalar"
53 // Allow forcing all i64 operations with equivalent SIMD instructions to use
54 // them. For stress-testing the transformation function.
56 TransformAll("aarch64-simd-scalar-force-all",
57 cl::desc("Force use of AdvSIMD scalar instructions everywhere"),
58 cl::init(false), cl::Hidden);
60 STATISTIC(NumScalarInsnsUsed, "Number of scalar instructions used");
61 STATISTIC(NumCopiesDeleted, "Number of cross-class copies deleted");
62 STATISTIC(NumCopiesInserted, "Number of cross-class copies inserted");
64 #define AARCH64_ADVSIMD_NAME "AdvSIMD Scalar Operation Optimization"
67 class AArch64AdvSIMDScalar : public MachineFunctionPass {
68 MachineRegisterInfo *MRI;
69 const TargetInstrInfo *TII;
72 // isProfitableToTransform - Predicate function to determine whether an
73 // instruction should be transformed to its equivalent AdvSIMD scalar
74 // instruction. "add Xd, Xn, Xm" ==> "add Dd, Da, Db", for example.
75 bool isProfitableToTransform(const MachineInstr &MI) const;
77 // transformInstruction - Perform the transformation of an instruction
78 // to its equivalant AdvSIMD scalar instruction. Update inputs and outputs
79 // to be the correct register class, minimizing cross-class copies.
80 void transformInstruction(MachineInstr &MI);
82 // processMachineBasicBlock - Main optimzation loop.
83 bool processMachineBasicBlock(MachineBasicBlock *MBB);
86 static char ID; // Pass identification, replacement for typeid.
87 explicit AArch64AdvSIMDScalar() : MachineFunctionPass(ID) {
88 initializeAArch64AdvSIMDScalarPass(*PassRegistry::getPassRegistry());
91 bool runOnMachineFunction(MachineFunction &F) override;
93 StringRef getPassName() const override { return AARCH64_ADVSIMD_NAME; }
95 void getAnalysisUsage(AnalysisUsage &AU) const override {
97 MachineFunctionPass::getAnalysisUsage(AU);
100 char AArch64AdvSIMDScalar::ID = 0;
101 } // end anonymous namespace
103 INITIALIZE_PASS(AArch64AdvSIMDScalar, "aarch64-simd-scalar",
104 AARCH64_ADVSIMD_NAME, false, false)
106 static bool isGPR64(unsigned Reg, unsigned SubReg,
107 const MachineRegisterInfo *MRI) {
110 if (TargetRegisterInfo::isVirtualRegister(Reg))
111 return MRI->getRegClass(Reg)->hasSuperClassEq(&AArch64::GPR64RegClass);
112 return AArch64::GPR64RegClass.contains(Reg);
115 static bool isFPR64(unsigned Reg, unsigned SubReg,
116 const MachineRegisterInfo *MRI) {
117 if (TargetRegisterInfo::isVirtualRegister(Reg))
118 return (MRI->getRegClass(Reg)->hasSuperClassEq(&AArch64::FPR64RegClass) &&
120 (MRI->getRegClass(Reg)->hasSuperClassEq(&AArch64::FPR128RegClass) &&
121 SubReg == AArch64::dsub);
122 // Physical register references just check the register class directly.
123 return (AArch64::FPR64RegClass.contains(Reg) && SubReg == 0) ||
124 (AArch64::FPR128RegClass.contains(Reg) && SubReg == AArch64::dsub);
127 // getSrcFromCopy - Get the original source register for a GPR64 <--> FPR64
128 // copy instruction. Return zero_reg if the instruction is not a copy.
129 static MachineOperand *getSrcFromCopy(MachineInstr *MI,
130 const MachineRegisterInfo *MRI,
133 // The "FMOV Xd, Dn" instruction is the typical form.
134 if (MI->getOpcode() == AArch64::FMOVDXr ||
135 MI->getOpcode() == AArch64::FMOVXDr)
136 return &MI->getOperand(1);
137 // A lane zero extract "UMOV.d Xd, Vn[0]" is equivalent. We shouldn't see
138 // these at this stage, but it's easy to check for.
139 if (MI->getOpcode() == AArch64::UMOVvi64 && MI->getOperand(2).getImm() == 0) {
140 SubReg = AArch64::dsub;
141 return &MI->getOperand(1);
143 // Or just a plain COPY instruction. This can be directly to/from FPR64,
144 // or it can be a dsub subreg reference to an FPR128.
145 if (MI->getOpcode() == AArch64::COPY) {
146 if (isFPR64(MI->getOperand(0).getReg(), MI->getOperand(0).getSubReg(),
148 isGPR64(MI->getOperand(1).getReg(), MI->getOperand(1).getSubReg(), MRI))
149 return &MI->getOperand(1);
150 if (isGPR64(MI->getOperand(0).getReg(), MI->getOperand(0).getSubReg(),
152 isFPR64(MI->getOperand(1).getReg(), MI->getOperand(1).getSubReg(),
154 SubReg = MI->getOperand(1).getSubReg();
155 return &MI->getOperand(1);
159 // Otherwise, this is some other kind of instruction.
163 // getTransformOpcode - For any opcode for which there is an AdvSIMD equivalent
164 // that we're considering transforming to, return that AdvSIMD opcode. For all
165 // others, return the original opcode.
166 static unsigned getTransformOpcode(unsigned Opc) {
170 // FIXME: Lots more possibilities.
171 case AArch64::ADDXrr:
172 return AArch64::ADDv1i64;
173 case AArch64::SUBXrr:
174 return AArch64::SUBv1i64;
175 case AArch64::ANDXrr:
176 return AArch64::ANDv8i8;
177 case AArch64::EORXrr:
178 return AArch64::EORv8i8;
179 case AArch64::ORRXrr:
180 return AArch64::ORRv8i8;
182 // No AdvSIMD equivalent, so just return the original opcode.
186 static bool isTransformable(const MachineInstr &MI) {
187 unsigned Opc = MI.getOpcode();
188 return Opc != getTransformOpcode(Opc);
191 // isProfitableToTransform - Predicate function to determine whether an
192 // instruction should be transformed to its equivalent AdvSIMD scalar
193 // instruction. "add Xd, Xn, Xm" ==> "add Dd, Da, Db", for example.
194 bool AArch64AdvSIMDScalar::isProfitableToTransform(
195 const MachineInstr &MI) const {
196 // If this instruction isn't eligible to be transformed (no SIMD equivalent),
197 // early exit since that's the common case.
198 if (!isTransformable(MI))
201 // Count the number of copies we'll need to add and approximate the number
202 // of copies that a transform will enable us to remove.
203 unsigned NumNewCopies = 3;
204 unsigned NumRemovableCopies = 0;
206 unsigned OrigSrc0 = MI.getOperand(1).getReg();
207 unsigned OrigSrc1 = MI.getOperand(2).getReg();
210 if (!MRI->def_empty(OrigSrc0)) {
211 MachineRegisterInfo::def_instr_iterator Def =
212 MRI->def_instr_begin(OrigSrc0);
213 assert(std::next(Def) == MRI->def_instr_end() && "Multiple def in SSA!");
214 MachineOperand *MOSrc0 = getSrcFromCopy(&*Def, MRI, SubReg0);
215 // If the source was from a copy, we don't need to insert a new copy.
218 // If there are no other users of the original source, we can delete
220 if (MOSrc0 && MRI->hasOneNonDBGUse(OrigSrc0))
221 ++NumRemovableCopies;
223 if (!MRI->def_empty(OrigSrc1)) {
224 MachineRegisterInfo::def_instr_iterator Def =
225 MRI->def_instr_begin(OrigSrc1);
226 assert(std::next(Def) == MRI->def_instr_end() && "Multiple def in SSA!");
227 MachineOperand *MOSrc1 = getSrcFromCopy(&*Def, MRI, SubReg1);
230 // If there are no other users of the original source, we can delete
232 if (MOSrc1 && MRI->hasOneNonDBGUse(OrigSrc1))
233 ++NumRemovableCopies;
236 // If any of the uses of the original instructions is a cross class copy,
237 // that's a copy that will be removable if we transform. Likewise, if
238 // any of the uses is a transformable instruction, it's likely the tranforms
239 // will chain, enabling us to save a copy there, too. This is an aggressive
240 // heuristic that approximates the graph based cost analysis described above.
241 unsigned Dst = MI.getOperand(0).getReg();
242 bool AllUsesAreCopies = true;
243 for (MachineRegisterInfo::use_instr_nodbg_iterator
244 Use = MRI->use_instr_nodbg_begin(Dst),
245 E = MRI->use_instr_nodbg_end();
248 if (getSrcFromCopy(&*Use, MRI, SubReg) || isTransformable(*Use))
249 ++NumRemovableCopies;
250 // If the use is an INSERT_SUBREG, that's still something that can
251 // directly use the FPR64, so we don't invalidate AllUsesAreCopies. It's
252 // preferable to have it use the FPR64 in most cases, as if the source
253 // vector is an IMPLICIT_DEF, the INSERT_SUBREG just goes away entirely.
254 // Ditto for a lane insert.
255 else if (Use->getOpcode() == AArch64::INSERT_SUBREG ||
256 Use->getOpcode() == AArch64::INSvi64gpr)
259 AllUsesAreCopies = false;
261 // If all of the uses of the original destination register are copies to
262 // FPR64, then we won't end up having a new copy back to GPR64 either.
263 if (AllUsesAreCopies)
266 // If a transform will not increase the number of cross-class copies required,
268 if (NumNewCopies <= NumRemovableCopies)
271 // Finally, even if we otherwise wouldn't transform, check if we're forcing
272 // transformation of everything.
276 static MachineInstr *insertCopy(const TargetInstrInfo *TII, MachineInstr &MI,
277 unsigned Dst, unsigned Src, bool IsKill) {
278 MachineInstrBuilder MIB = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
279 TII->get(AArch64::COPY), Dst)
280 .addReg(Src, getKillRegState(IsKill));
281 DEBUG(dbgs() << " adding copy: " << *MIB);
286 // transformInstruction - Perform the transformation of an instruction
287 // to its equivalant AdvSIMD scalar instruction. Update inputs and outputs
288 // to be the correct register class, minimizing cross-class copies.
289 void AArch64AdvSIMDScalar::transformInstruction(MachineInstr &MI) {
290 DEBUG(dbgs() << "Scalar transform: " << MI);
292 MachineBasicBlock *MBB = MI.getParent();
293 unsigned OldOpc = MI.getOpcode();
294 unsigned NewOpc = getTransformOpcode(OldOpc);
295 assert(OldOpc != NewOpc && "transform an instruction to itself?!");
297 // Check if we need a copy for the source registers.
298 unsigned OrigSrc0 = MI.getOperand(1).getReg();
299 unsigned OrigSrc1 = MI.getOperand(2).getReg();
300 unsigned Src0 = 0, SubReg0;
301 unsigned Src1 = 0, SubReg1;
302 bool KillSrc0 = false, KillSrc1 = false;
303 if (!MRI->def_empty(OrigSrc0)) {
304 MachineRegisterInfo::def_instr_iterator Def =
305 MRI->def_instr_begin(OrigSrc0);
306 assert(std::next(Def) == MRI->def_instr_end() && "Multiple def in SSA!");
307 MachineOperand *MOSrc0 = getSrcFromCopy(&*Def, MRI, SubReg0);
308 // If there are no other users of the original source, we can delete
311 Src0 = MOSrc0->getReg();
312 KillSrc0 = MOSrc0->isKill();
313 // Src0 is going to be reused, thus, it cannot be killed anymore.
314 MOSrc0->setIsKill(false);
315 if (MRI->hasOneNonDBGUse(OrigSrc0)) {
316 assert(MOSrc0 && "Can't delete copy w/o a valid original source!");
317 Def->eraseFromParent();
322 if (!MRI->def_empty(OrigSrc1)) {
323 MachineRegisterInfo::def_instr_iterator Def =
324 MRI->def_instr_begin(OrigSrc1);
325 assert(std::next(Def) == MRI->def_instr_end() && "Multiple def in SSA!");
326 MachineOperand *MOSrc1 = getSrcFromCopy(&*Def, MRI, SubReg1);
327 // If there are no other users of the original source, we can delete
330 Src1 = MOSrc1->getReg();
331 KillSrc1 = MOSrc1->isKill();
332 // Src0 is going to be reused, thus, it cannot be killed anymore.
333 MOSrc1->setIsKill(false);
334 if (MRI->hasOneNonDBGUse(OrigSrc1)) {
335 assert(MOSrc1 && "Can't delete copy w/o a valid original source!");
336 Def->eraseFromParent();
341 // If we weren't able to reference the original source directly, create a
345 Src0 = MRI->createVirtualRegister(&AArch64::FPR64RegClass);
346 insertCopy(TII, MI, Src0, OrigSrc0, KillSrc0);
351 Src1 = MRI->createVirtualRegister(&AArch64::FPR64RegClass);
352 insertCopy(TII, MI, Src1, OrigSrc1, KillSrc1);
356 // Create a vreg for the destination.
357 // FIXME: No need to do this if the ultimate user expects an FPR64.
358 // Check for that and avoid the copy if possible.
359 unsigned Dst = MRI->createVirtualRegister(&AArch64::FPR64RegClass);
361 // For now, all of the new instructions have the same simple three-register
362 // form, so no need to special case based on what instruction we're
364 BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(NewOpc), Dst)
365 .addReg(Src0, getKillRegState(KillSrc0), SubReg0)
366 .addReg(Src1, getKillRegState(KillSrc1), SubReg1);
368 // Now copy the result back out to a GPR.
369 // FIXME: Try to avoid this if all uses could actually just use the FPR64
371 insertCopy(TII, MI, MI.getOperand(0).getReg(), Dst, true);
373 // Erase the old instruction.
374 MI.eraseFromParent();
376 ++NumScalarInsnsUsed;
379 // processMachineBasicBlock - Main optimzation loop.
380 bool AArch64AdvSIMDScalar::processMachineBasicBlock(MachineBasicBlock *MBB) {
381 bool Changed = false;
382 for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;) {
383 MachineInstr &MI = *I++;
384 if (isProfitableToTransform(MI)) {
385 transformInstruction(MI);
392 // runOnMachineFunction - Pass entry point from PassManager.
393 bool AArch64AdvSIMDScalar::runOnMachineFunction(MachineFunction &mf) {
394 bool Changed = false;
395 DEBUG(dbgs() << "***** AArch64AdvSIMDScalar *****\n");
397 if (skipFunction(*mf.getFunction()))
400 MRI = &mf.getRegInfo();
401 TII = mf.getSubtarget().getInstrInfo();
403 // Just check things on a one-block-at-a-time basis.
404 for (MachineFunction::iterator I = mf.begin(), E = mf.end(); I != E; ++I)
405 if (processMachineBasicBlock(&*I))
410 // createAArch64AdvSIMDScalar - Factory function used by AArch64TargetMachine
411 // to add the pass to the PassManager.
412 FunctionPass *llvm::createAArch64AdvSIMDScalar() {
413 return new AArch64AdvSIMDScalar();