1 //===-- NVPTXTargetMachine.cpp - Define TargetMachine for NVPTX -----------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // Top-level implementation for the NVPTX target.
12 //===----------------------------------------------------------------------===//
14 #include "NVPTXTargetMachine.h"
15 #include "MCTargetDesc/NVPTXMCAsmInfo.h"
17 #include "NVPTXAllocaHoisting.h"
18 #include "NVPTXLowerAggrCopies.h"
19 #include "NVPTXTargetObjectFile.h"
20 #include "NVPTXTargetTransformInfo.h"
21 #include "llvm/Analysis/Passes.h"
22 #include "llvm/CodeGen/AsmPrinter.h"
23 #include "llvm/CodeGen/MachineModuleInfo.h"
24 #include "llvm/CodeGen/Passes.h"
25 #include "llvm/CodeGen/TargetPassConfig.h"
26 #include "llvm/IR/DataLayout.h"
27 #include "llvm/IR/IRPrintingPasses.h"
28 #include "llvm/IR/LegacyPassManager.h"
29 #include "llvm/IR/Verifier.h"
30 #include "llvm/MC/MCAsmInfo.h"
31 #include "llvm/MC/MCInstrInfo.h"
32 #include "llvm/MC/MCStreamer.h"
33 #include "llvm/MC/MCSubtargetInfo.h"
34 #include "llvm/Support/CommandLine.h"
35 #include "llvm/Support/Debug.h"
36 #include "llvm/Support/FormattedStream.h"
37 #include "llvm/Support/TargetRegistry.h"
38 #include "llvm/Support/raw_ostream.h"
39 #include "llvm/Target/TargetInstrInfo.h"
40 #include "llvm/Target/TargetLowering.h"
41 #include "llvm/Target/TargetLoweringObjectFile.h"
42 #include "llvm/Target/TargetMachine.h"
43 #include "llvm/Target/TargetOptions.h"
44 #include "llvm/Target/TargetRegisterInfo.h"
45 #include "llvm/Target/TargetSubtargetInfo.h"
46 #include "llvm/Transforms/Scalar.h"
47 #include "llvm/Transforms/Scalar/GVN.h"
48 #include "llvm/Transforms/Vectorize.h"
52 // LSV is still relatively new; this switch lets us turn it off in case we
53 // encounter (or suspect) a bug.
55 DisableLoadStoreVectorizer("disable-nvptx-load-store-vectorizer",
56 cl::desc("Disable load/store vectorizer"),
57 cl::init(false), cl::Hidden);
60 void initializeNVVMIntrRangePass(PassRegistry&);
61 void initializeNVVMReflectPass(PassRegistry&);
62 void initializeGenericToNVVMPass(PassRegistry&);
63 void initializeNVPTXAllocaHoistingPass(PassRegistry &);
64 void initializeNVPTXAssignValidGlobalNamesPass(PassRegistry&);
65 void initializeNVPTXInferAddressSpacesPass(PassRegistry &);
66 void initializeNVPTXLowerAggrCopiesPass(PassRegistry &);
67 void initializeNVPTXLowerArgsPass(PassRegistry &);
68 void initializeNVPTXLowerAllocaPass(PassRegistry &);
71 extern "C" void LLVMInitializeNVPTXTarget() {
72 // Register the target.
73 RegisterTargetMachine<NVPTXTargetMachine32> X(getTheNVPTXTarget32());
74 RegisterTargetMachine<NVPTXTargetMachine64> Y(getTheNVPTXTarget64());
76 // FIXME: This pass is really intended to be invoked during IR optimization,
77 // but it's very NVPTX-specific.
78 PassRegistry &PR = *PassRegistry::getPassRegistry();
79 initializeNVVMReflectPass(PR);
80 initializeNVVMIntrRangePass(PR);
81 initializeGenericToNVVMPass(PR);
82 initializeNVPTXAllocaHoistingPass(PR);
83 initializeNVPTXAssignValidGlobalNamesPass(PR);
84 initializeNVPTXInferAddressSpacesPass(PR);
85 initializeNVPTXLowerArgsPass(PR);
86 initializeNVPTXLowerAllocaPass(PR);
87 initializeNVPTXLowerAggrCopiesPass(PR);
90 static std::string computeDataLayout(bool is64Bit) {
91 std::string Ret = "e";
96 Ret += "-i64:64-v16:16-v32:32-n16:32:64";
101 NVPTXTargetMachine::NVPTXTargetMachine(const Target &T, const Triple &TT,
102 StringRef CPU, StringRef FS,
103 const TargetOptions &Options,
104 Optional<Reloc::Model> RM,
106 CodeGenOpt::Level OL, bool is64bit)
107 // The pic relocation model is used regardless of what the client has
108 // specified, as it is the only relocation model currently supported.
109 : LLVMTargetMachine(T, computeDataLayout(is64bit), TT, CPU, FS, Options,
110 Reloc::PIC_, CM, OL),
112 TLOF(make_unique<NVPTXTargetObjectFile>()),
113 Subtarget(TT, CPU, FS, *this) {
114 if (TT.getOS() == Triple::NVCL)
115 drvInterface = NVPTX::NVCL;
117 drvInterface = NVPTX::CUDA;
121 NVPTXTargetMachine::~NVPTXTargetMachine() {}
123 void NVPTXTargetMachine32::anchor() {}
125 NVPTXTargetMachine32::NVPTXTargetMachine32(const Target &T, const Triple &TT,
126 StringRef CPU, StringRef FS,
127 const TargetOptions &Options,
128 Optional<Reloc::Model> RM,
130 CodeGenOpt::Level OL)
131 : NVPTXTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {}
133 void NVPTXTargetMachine64::anchor() {}
135 NVPTXTargetMachine64::NVPTXTargetMachine64(const Target &T, const Triple &TT,
136 StringRef CPU, StringRef FS,
137 const TargetOptions &Options,
138 Optional<Reloc::Model> RM,
140 CodeGenOpt::Level OL)
141 : NVPTXTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {}
144 class NVPTXPassConfig : public TargetPassConfig {
146 NVPTXPassConfig(NVPTXTargetMachine *TM, PassManagerBase &PM)
147 : TargetPassConfig(TM, PM) {}
149 NVPTXTargetMachine &getNVPTXTargetMachine() const {
150 return getTM<NVPTXTargetMachine>();
153 void addIRPasses() override;
154 bool addInstSelector() override;
155 void addPostRegAlloc() override;
156 void addMachineSSAOptimization() override;
158 FunctionPass *createTargetRegisterAllocator(bool) override;
159 void addFastRegAlloc(FunctionPass *RegAllocPass) override;
160 void addOptimizedRegAlloc(FunctionPass *RegAllocPass) override;
163 // If the opt level is aggressive, add GVN; otherwise, add EarlyCSE. This
164 // function is only called in opt mode.
165 void addEarlyCSEOrGVNPass();
167 // Add passes that propagate special memory spaces.
168 void addAddressSpaceInferencePasses();
170 // Add passes that perform straight-line scalar optimizations.
171 void addStraightLineScalarOptimizationPasses();
173 } // end anonymous namespace
175 TargetPassConfig *NVPTXTargetMachine::createPassConfig(PassManagerBase &PM) {
176 return new NVPTXPassConfig(this, PM);
179 void NVPTXTargetMachine::addEarlyAsPossiblePasses(PassManagerBase &PM) {
180 PM.add(createNVVMReflectPass());
181 PM.add(createNVVMIntrRangePass(Subtarget.getSmVersion()));
184 TargetIRAnalysis NVPTXTargetMachine::getTargetIRAnalysis() {
185 return TargetIRAnalysis([this](const Function &F) {
186 return TargetTransformInfo(NVPTXTTIImpl(this, F));
190 void NVPTXPassConfig::addEarlyCSEOrGVNPass() {
191 if (getOptLevel() == CodeGenOpt::Aggressive)
192 addPass(createGVNPass());
194 addPass(createEarlyCSEPass());
197 void NVPTXPassConfig::addAddressSpaceInferencePasses() {
198 // NVPTXLowerArgs emits alloca for byval parameters which can often
199 // be eliminated by SROA.
200 addPass(createSROAPass());
201 addPass(createNVPTXLowerAllocaPass());
202 addPass(createNVPTXInferAddressSpacesPass());
205 void NVPTXPassConfig::addStraightLineScalarOptimizationPasses() {
206 addPass(createSeparateConstOffsetFromGEPPass());
207 addPass(createSpeculativeExecutionPass());
208 // ReassociateGEPs exposes more opportunites for SLSR. See
209 // the example in reassociate-geps-and-slsr.ll.
210 addPass(createStraightLineStrengthReducePass());
211 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
212 // EarlyCSE can reuse. GVN generates significantly better code than EarlyCSE
213 // for some of our benchmarks.
214 addEarlyCSEOrGVNPass();
215 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
216 addPass(createNaryReassociatePass());
217 // NaryReassociate on GEPs creates redundant common expressions, so run
218 // EarlyCSE after it.
219 addPass(createEarlyCSEPass());
222 void NVPTXPassConfig::addIRPasses() {
223 // The following passes are known to not play well with virtual regs hanging
224 // around after register allocation (which in our case, is *all* registers).
225 // We explicitly disable them here. We do, however, need some functionality
226 // of the PrologEpilogCodeInserter pass, so we emulate that behavior in the
227 // NVPTXPrologEpilog pass (see NVPTXPrologEpilogPass.cpp).
228 disablePass(&PrologEpilogCodeInserterID);
229 disablePass(&MachineCopyPropagationID);
230 disablePass(&TailDuplicateID);
231 disablePass(&StackMapLivenessID);
232 disablePass(&LiveDebugValuesID);
233 disablePass(&PostRASchedulerID);
234 disablePass(&FuncletLayoutID);
235 disablePass(&PatchableFunctionID);
237 // NVVMReflectPass is added in addEarlyAsPossiblePasses, so hopefully running
238 // it here does nothing. But since we need it for correctness when lowering
239 // to NVPTX, run it here too, in case whoever built our pass pipeline didn't
240 // call addEarlyAsPossiblePasses.
241 addPass(createNVVMReflectPass());
243 if (getOptLevel() != CodeGenOpt::None)
244 addPass(createNVPTXImageOptimizerPass());
245 addPass(createNVPTXAssignValidGlobalNamesPass());
246 addPass(createGenericToNVVMPass());
248 // NVPTXLowerArgs is required for correctness and should be run right
249 // before the address space inference passes.
250 addPass(createNVPTXLowerArgsPass(&getNVPTXTargetMachine()));
251 if (getOptLevel() != CodeGenOpt::None) {
252 addAddressSpaceInferencePasses();
253 if (!DisableLoadStoreVectorizer)
254 addPass(createLoadStoreVectorizerPass());
255 addStraightLineScalarOptimizationPasses();
258 // === LSR and other generic IR passes ===
259 TargetPassConfig::addIRPasses();
260 // EarlyCSE is not always strong enough to clean up what LSR produces. For
261 // example, GVN can combine
268 // %0 = shl nsw %a, 2
271 // but EarlyCSE can do neither of them.
272 if (getOptLevel() != CodeGenOpt::None)
273 addEarlyCSEOrGVNPass();
276 bool NVPTXPassConfig::addInstSelector() {
277 const NVPTXSubtarget &ST = *getTM<NVPTXTargetMachine>().getSubtargetImpl();
279 addPass(createLowerAggrCopies());
280 addPass(createAllocaHoisting());
281 addPass(createNVPTXISelDag(getNVPTXTargetMachine(), getOptLevel()));
283 if (!ST.hasImageHandles())
284 addPass(createNVPTXReplaceImageHandlesPass());
289 void NVPTXPassConfig::addPostRegAlloc() {
290 addPass(createNVPTXPrologEpilogPass(), false);
291 if (getOptLevel() != CodeGenOpt::None) {
292 // NVPTXPrologEpilogPass calculates frame object offset and replace frame
293 // index with VRFrame register. NVPTXPeephole need to be run after that and
294 // will replace VRFrame with VRFrameLocal when possible.
295 addPass(createNVPTXPeephole());
299 FunctionPass *NVPTXPassConfig::createTargetRegisterAllocator(bool) {
300 return nullptr; // No reg alloc
303 void NVPTXPassConfig::addFastRegAlloc(FunctionPass *RegAllocPass) {
304 assert(!RegAllocPass && "NVPTX uses no regalloc!");
305 addPass(&PHIEliminationID);
306 addPass(&TwoAddressInstructionPassID);
309 void NVPTXPassConfig::addOptimizedRegAlloc(FunctionPass *RegAllocPass) {
310 assert(!RegAllocPass && "NVPTX uses no regalloc!");
312 addPass(&ProcessImplicitDefsID);
313 addPass(&LiveVariablesID);
314 addPass(&MachineLoopInfoID);
315 addPass(&PHIEliminationID);
317 addPass(&TwoAddressInstructionPassID);
318 addPass(&RegisterCoalescerID);
320 // PreRA instruction scheduling.
321 if (addPass(&MachineSchedulerID))
322 printAndVerify("After Machine Scheduling");
325 addPass(&StackSlotColoringID);
327 // FIXME: Needs physical registers
328 //addPass(&PostRAMachineLICMID);
330 printAndVerify("After StackSlotColoring");
333 void NVPTXPassConfig::addMachineSSAOptimization() {
334 // Pre-ra tail duplication.
335 if (addPass(&EarlyTailDuplicateID))
336 printAndVerify("After Pre-RegAlloc TailDuplicate");
338 // Optimize PHIs before DCE: removing dead PHI cycles may make more
339 // instructions dead.
340 addPass(&OptimizePHIsID);
342 // This pass merges large allocas. StackSlotColoring is a different pass
343 // which merges spill slots.
344 addPass(&StackColoringID);
346 // If the target requests it, assign local variables to stack slots relative
347 // to one another and simplify frame index references where possible.
348 addPass(&LocalStackSlotAllocationID);
350 // With optimization, dead code should already be eliminated. However
351 // there is one known exception: lowered code for arguments that are only
352 // used by tail calls, where the tail calls reuse the incoming stack
353 // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
354 addPass(&DeadMachineInstructionElimID);
355 printAndVerify("After codegen DCE pass");
357 // Allow targets to insert passes that improve instruction level parallelism,
358 // like if-conversion. Such passes will typically need dominator trees and
359 // loop info, just like LICM and CSE below.
361 printAndVerify("After ILP optimizations");
363 addPass(&MachineLICMID);
364 addPass(&MachineCSEID);
366 addPass(&MachineSinkingID);
367 printAndVerify("After Machine LICM, CSE and Sinking passes");
369 addPass(&PeepholeOptimizerID);
370 printAndVerify("After codegen peephole optimization pass");