1 //===- PassManagerBuilder.cpp - Build Standard Pass -----------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the PassManagerBuilder class, which is used to set up a
11 // "standard" optimization sequence suitable for languages like C and C++.
13 //===----------------------------------------------------------------------===//
15 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
16 #include "llvm-c/Transforms/PassManagerBuilder.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/Analysis/BasicAliasAnalysis.h"
19 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
20 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
21 #include "llvm/Analysis/GlobalsModRef.h"
22 #include "llvm/Analysis/InlineCost.h"
23 #include "llvm/Analysis/Passes.h"
24 #include "llvm/Analysis/ScopedNoAliasAA.h"
25 #include "llvm/Analysis/TargetLibraryInfo.h"
26 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
27 #include "llvm/IR/DataLayout.h"
28 #include "llvm/IR/LegacyPassManager.h"
29 #include "llvm/IR/ModuleSummaryIndex.h"
30 #include "llvm/IR/Verifier.h"
31 #include "llvm/Support/CommandLine.h"
32 #include "llvm/Support/ManagedStatic.h"
33 #include "llvm/Target/TargetMachine.h"
34 #include "llvm/Transforms/IPO.h"
35 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
36 #include "llvm/Transforms/IPO/FunctionAttrs.h"
37 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
38 #include "llvm/Transforms/Instrumentation.h"
39 #include "llvm/Transforms/Scalar.h"
40 #include "llvm/Transforms/Scalar/GVN.h"
41 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
42 #include "llvm/Transforms/Vectorize.h"
47 RunPartialInlining("enable-partial-inlining", cl::init(false), cl::Hidden,
48 cl::ZeroOrMore, cl::desc("Run Partial inlinining pass"));
51 RunLoopVectorization("vectorize-loops", cl::Hidden,
52 cl::desc("Run the Loop vectorization passes"));
55 RunSLPVectorization("vectorize-slp", cl::Hidden,
56 cl::desc("Run the SLP vectorization passes"));
59 RunBBVectorization("vectorize-slp-aggressive", cl::Hidden,
60 cl::desc("Run the BB vectorization passes"));
63 UseGVNAfterVectorization("use-gvn-after-vectorization",
64 cl::init(false), cl::Hidden,
65 cl::desc("Run GVN instead of Early CSE after vectorization passes"));
67 static cl::opt<bool> ExtraVectorizerPasses(
68 "extra-vectorizer-passes", cl::init(false), cl::Hidden,
69 cl::desc("Run cleanup optimization passes after vectorization."));
72 RunLoopRerolling("reroll-loops", cl::Hidden,
73 cl::desc("Run the loop rerolling pass"));
75 static cl::opt<bool> RunLoadCombine("combine-loads", cl::init(false),
77 cl::desc("Run the load combining pass"));
79 static cl::opt<bool> RunNewGVN("enable-newgvn", cl::init(false), cl::Hidden,
80 cl::desc("Run the NewGVN pass"));
83 RunSLPAfterLoopVectorization("run-slp-after-loop-vectorization",
84 cl::init(true), cl::Hidden,
85 cl::desc("Run the SLP vectorizer (and BB vectorizer) after the Loop "
86 "vectorizer instead of before"));
88 // Experimental option to use CFL-AA
89 enum class CFLAAType { None, Steensgaard, Andersen, Both };
90 static cl::opt<CFLAAType>
91 UseCFLAA("use-cfl-aa", cl::init(CFLAAType::None), cl::Hidden,
92 cl::desc("Enable the new, experimental CFL alias analysis"),
93 cl::values(clEnumValN(CFLAAType::None, "none", "Disable CFL-AA"),
94 clEnumValN(CFLAAType::Steensgaard, "steens",
95 "Enable unification-based CFL-AA"),
96 clEnumValN(CFLAAType::Andersen, "anders",
97 "Enable inclusion-based CFL-AA"),
98 clEnumValN(CFLAAType::Both, "both",
99 "Enable both variants of CFL-AA")));
101 static cl::opt<bool> EnableLoopInterchange(
102 "enable-loopinterchange", cl::init(false), cl::Hidden,
103 cl::desc("Enable the new, experimental LoopInterchange Pass"));
105 static cl::opt<bool> EnableNonLTOGlobalsModRef(
106 "enable-non-lto-gmr", cl::init(true), cl::Hidden,
108 "Enable the GlobalsModRef AliasAnalysis outside of the LTO pipeline."));
110 static cl::opt<bool> EnableLoopLoadElim(
111 "enable-loop-load-elim", cl::init(true), cl::Hidden,
112 cl::desc("Enable the LoopLoadElimination Pass"));
115 EnablePrepareForThinLTO("prepare-for-thinlto", cl::init(false), cl::Hidden,
116 cl::desc("Enable preparation for ThinLTO."));
118 static cl::opt<bool> RunPGOInstrGen(
119 "profile-generate", cl::init(false), cl::Hidden,
120 cl::desc("Enable PGO instrumentation."));
122 static cl::opt<std::string>
123 PGOOutputFile("profile-generate-file", cl::init(""), cl::Hidden,
124 cl::desc("Specify the path of profile data file."));
126 static cl::opt<std::string> RunPGOInstrUse(
127 "profile-use", cl::init(""), cl::Hidden, cl::value_desc("filename"),
128 cl::desc("Enable use phase of PGO instrumentation and specify the path "
129 "of profile data file"));
131 static cl::opt<bool> UseLoopVersioningLICM(
132 "enable-loop-versioning-licm", cl::init(false), cl::Hidden,
133 cl::desc("Enable the experimental Loop Versioning LICM pass"));
136 DisablePreInliner("disable-preinline", cl::init(false), cl::Hidden,
137 cl::desc("Disable pre-instrumentation inliner"));
139 static cl::opt<int> PreInlineThreshold(
140 "preinline-threshold", cl::Hidden, cl::init(75), cl::ZeroOrMore,
141 cl::desc("Control the amount of inlining in pre-instrumentation inliner "
144 static cl::opt<bool> EnableGVNHoist(
145 "enable-gvn-hoist", cl::init(false), cl::Hidden,
146 cl::desc("Enable the GVN hoisting pass (default = off)"));
149 DisableLibCallsShrinkWrap("disable-libcalls-shrinkwrap", cl::init(false),
151 cl::desc("Disable shrink-wrap library calls"));
154 EnableSimpleLoopUnswitch("enable-simple-loop-unswitch", cl::init(false),
156 cl::desc("Enable the simple loop unswitch pass."));
158 static cl::opt<bool> EnableGVNSink(
159 "enable-gvn-sink", cl::init(false), cl::Hidden,
160 cl::desc("Enable the GVN sinking pass (default = on)"));
162 PassManagerBuilder::PassManagerBuilder() {
165 LibraryInfo = nullptr;
167 DisableUnitAtATime = false;
168 DisableUnrollLoops = false;
169 BBVectorize = RunBBVectorization;
170 SLPVectorize = RunSLPVectorization;
171 LoopVectorize = RunLoopVectorization;
172 RerollLoops = RunLoopRerolling;
173 LoadCombine = RunLoadCombine;
175 DisableGVNLoadPRE = false;
177 VerifyOutput = false;
178 MergeFunctions = false;
179 PrepareForLTO = false;
180 EnablePGOInstrGen = RunPGOInstrGen;
181 PGOInstrGen = PGOOutputFile;
182 PGOInstrUse = RunPGOInstrUse;
183 PrepareForThinLTO = EnablePrepareForThinLTO;
184 PerformThinLTO = false;
185 DivergentTarget = false;
188 PassManagerBuilder::~PassManagerBuilder() {
193 /// Set of global extensions, automatically added as part of the standard set.
194 static ManagedStatic<SmallVector<std::pair<PassManagerBuilder::ExtensionPointTy,
195 PassManagerBuilder::ExtensionFn>, 8> > GlobalExtensions;
197 void PassManagerBuilder::addGlobalExtension(
198 PassManagerBuilder::ExtensionPointTy Ty,
199 PassManagerBuilder::ExtensionFn Fn) {
200 GlobalExtensions->push_back(std::make_pair(Ty, std::move(Fn)));
203 void PassManagerBuilder::addExtension(ExtensionPointTy Ty, ExtensionFn Fn) {
204 Extensions.push_back(std::make_pair(Ty, std::move(Fn)));
207 void PassManagerBuilder::addExtensionsToPM(ExtensionPointTy ETy,
208 legacy::PassManagerBase &PM) const {
209 for (unsigned i = 0, e = GlobalExtensions->size(); i != e; ++i)
210 if ((*GlobalExtensions)[i].first == ETy)
211 (*GlobalExtensions)[i].second(*this, PM);
212 for (unsigned i = 0, e = Extensions.size(); i != e; ++i)
213 if (Extensions[i].first == ETy)
214 Extensions[i].second(*this, PM);
217 void PassManagerBuilder::addInitialAliasAnalysisPasses(
218 legacy::PassManagerBase &PM) const {
220 case CFLAAType::Steensgaard:
221 PM.add(createCFLSteensAAWrapperPass());
223 case CFLAAType::Andersen:
224 PM.add(createCFLAndersAAWrapperPass());
226 case CFLAAType::Both:
227 PM.add(createCFLSteensAAWrapperPass());
228 PM.add(createCFLAndersAAWrapperPass());
234 // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
235 // BasicAliasAnalysis wins if they disagree. This is intended to help
236 // support "obvious" type-punning idioms.
237 PM.add(createTypeBasedAAWrapperPass());
238 PM.add(createScopedNoAliasAAWrapperPass());
241 void PassManagerBuilder::addInstructionCombiningPass(
242 legacy::PassManagerBase &PM) const {
243 bool ExpensiveCombines = OptLevel > 2;
244 PM.add(createInstructionCombiningPass(ExpensiveCombines));
247 void PassManagerBuilder::populateFunctionPassManager(
248 legacy::FunctionPassManager &FPM) {
249 addExtensionsToPM(EP_EarlyAsPossible, FPM);
251 // Add LibraryInfo if we have some.
253 FPM.add(new TargetLibraryInfoWrapperPass(*LibraryInfo));
255 if (OptLevel == 0) return;
257 addInitialAliasAnalysisPasses(FPM);
259 FPM.add(createCFGSimplificationPass());
260 FPM.add(createSROAPass());
261 FPM.add(createEarlyCSEPass());
262 FPM.add(createLowerExpectIntrinsicPass());
265 // Do PGO instrumentation generation or use pass as the option specified.
266 void PassManagerBuilder::addPGOInstrPasses(legacy::PassManagerBase &MPM) {
267 if (!EnablePGOInstrGen && PGOInstrUse.empty())
269 // Perform the preinline and cleanup passes for O1 and above.
270 // And avoid doing them if optimizing for size.
271 if (OptLevel > 0 && SizeLevel == 0 && !DisablePreInliner) {
272 // Create preinline pass. We construct an InlineParams object and specify
273 // the threshold here to avoid the command line options of the regular
274 // inliner to influence pre-inlining. The only fields of InlineParams we
275 // care about are DefaultThreshold and HintThreshold.
277 IP.DefaultThreshold = PreInlineThreshold;
278 // FIXME: The hint threshold has the same value used by the regular inliner.
279 // This should probably be lowered after performance testing.
280 IP.HintThreshold = 325;
282 MPM.add(createFunctionInliningPass(IP));
283 MPM.add(createSROAPass());
284 MPM.add(createEarlyCSEPass()); // Catch trivial redundancies
285 MPM.add(createCFGSimplificationPass()); // Merge & remove BBs
286 MPM.add(createInstructionCombiningPass()); // Combine silly seq's
287 addExtensionsToPM(EP_Peephole, MPM);
289 if (EnablePGOInstrGen) {
290 MPM.add(createPGOInstrumentationGenLegacyPass());
291 // Add the profile lowering pass.
292 InstrProfOptions Options;
293 if (!PGOInstrGen.empty())
294 Options.InstrProfileOutput = PGOInstrGen;
295 MPM.add(createInstrProfilingLegacyPass(Options));
297 if (!PGOInstrUse.empty())
298 MPM.add(createPGOInstrumentationUseLegacyPass(PGOInstrUse));
299 // Indirect call promotion that promotes intra-module targets only.
300 // For ThinLTO this is done earlier due to interactions with globalopt
301 // for imported functions. We don't run this at -O0.
304 createPGOIndirectCallPromotionLegacyPass(false, !PGOSampleUse.empty()));
306 void PassManagerBuilder::addFunctionSimplificationPasses(
307 legacy::PassManagerBase &MPM) {
308 // Start of function pass.
309 // Break up aggregate allocas, using SSAUpdater.
310 MPM.add(createSROAPass());
311 MPM.add(createEarlyCSEPass()); // Catch trivial redundancies
313 MPM.add(createGVNHoistPass());
315 MPM.add(createGVNSinkPass());
316 MPM.add(createCFGSimplificationPass());
319 // Speculative execution if the target has divergent branches; otherwise nop.
320 MPM.add(createSpeculativeExecutionIfHasBranchDivergencePass());
321 MPM.add(createJumpThreadingPass()); // Thread jumps.
322 MPM.add(createCorrelatedValuePropagationPass()); // Propagate conditionals
323 MPM.add(createCFGSimplificationPass()); // Merge & remove BBs
324 // Combine silly seq's
325 addInstructionCombiningPass(MPM);
326 if (SizeLevel == 0 && !DisableLibCallsShrinkWrap)
327 MPM.add(createLibCallsShrinkWrapPass());
328 addExtensionsToPM(EP_Peephole, MPM);
330 // Optimize memory intrinsic calls based on the profiled size information.
332 MPM.add(createPGOMemOPSizeOptLegacyPass());
334 MPM.add(createTailCallEliminationPass()); // Eliminate tail calls
335 MPM.add(createCFGSimplificationPass()); // Merge & remove BBs
336 MPM.add(createReassociatePass()); // Reassociate expressions
337 // Rotate Loop - disable header duplication at -Oz
338 MPM.add(createLoopRotatePass(SizeLevel == 2 ? 0 : -1));
339 MPM.add(createLICMPass()); // Hoist loop invariants
340 if (EnableSimpleLoopUnswitch)
341 MPM.add(createSimpleLoopUnswitchLegacyPass());
343 MPM.add(createLoopUnswitchPass(SizeLevel || OptLevel < 3, DivergentTarget));
344 MPM.add(createCFGSimplificationPass());
345 addInstructionCombiningPass(MPM);
346 MPM.add(createIndVarSimplifyPass()); // Canonicalize indvars
347 MPM.add(createLoopIdiomPass()); // Recognize idioms like memset.
348 addExtensionsToPM(EP_LateLoopOptimizations, MPM);
349 MPM.add(createLoopDeletionPass()); // Delete dead loops
351 if (EnableLoopInterchange) {
352 MPM.add(createLoopInterchangePass()); // Interchange loops
353 MPM.add(createCFGSimplificationPass());
355 if (!DisableUnrollLoops)
356 MPM.add(createSimpleLoopUnrollPass(OptLevel)); // Unroll small loops
357 addExtensionsToPM(EP_LoopOptimizerEnd, MPM);
360 MPM.add(createMergedLoadStoreMotionPass()); // Merge ld/st in diamonds
361 MPM.add(NewGVN ? createNewGVNPass()
362 : createGVNPass(DisableGVNLoadPRE)); // Remove redundancies
364 MPM.add(createMemCpyOptPass()); // Remove memcpy / form memset
365 MPM.add(createSCCPPass()); // Constant prop with SCCP
367 // Delete dead bit computations (instcombine runs after to fold away the dead
368 // computations, and then ADCE will run later to exploit any new DCE
369 // opportunities that creates).
370 MPM.add(createBitTrackingDCEPass()); // Delete dead bit computations
372 // Run instcombine after redundancy elimination to exploit opportunities
373 // opened up by them.
374 addInstructionCombiningPass(MPM);
375 addExtensionsToPM(EP_Peephole, MPM);
376 MPM.add(createJumpThreadingPass()); // Thread jumps
377 MPM.add(createCorrelatedValuePropagationPass());
378 MPM.add(createDeadStoreEliminationPass()); // Delete dead stores
379 MPM.add(createLICMPass());
381 addExtensionsToPM(EP_ScalarOptimizerLate, MPM);
384 MPM.add(createLoopRerollPass());
385 if (!RunSLPAfterLoopVectorization) {
387 MPM.add(createSLPVectorizerPass()); // Vectorize parallel scalar chains.
390 MPM.add(createBBVectorizePass());
391 addInstructionCombiningPass(MPM);
392 addExtensionsToPM(EP_Peephole, MPM);
393 if (OptLevel > 1 && UseGVNAfterVectorization)
396 : createGVNPass(DisableGVNLoadPRE)); // Remove redundancies
398 MPM.add(createEarlyCSEPass()); // Catch trivial redundancies
400 // BBVectorize may have significantly shortened a loop body; unroll again.
401 if (!DisableUnrollLoops)
402 MPM.add(createLoopUnrollPass(OptLevel));
407 MPM.add(createLoadCombinePass());
409 MPM.add(createAggressiveDCEPass()); // Delete dead instructions
410 MPM.add(createCFGSimplificationPass()); // Merge & remove BBs
411 // Clean up after everything.
412 addInstructionCombiningPass(MPM);
413 addExtensionsToPM(EP_Peephole, MPM);
416 void PassManagerBuilder::populateModulePassManager(
417 legacy::PassManagerBase &MPM) {
418 if (!PGOSampleUse.empty()) {
419 MPM.add(createPruneEHPass());
420 MPM.add(createSampleProfileLoaderPass(PGOSampleUse));
423 // Allow forcing function attributes as a debugging and tuning aid.
424 MPM.add(createForceFunctionAttrsLegacyPass());
426 // If all optimizations are disabled, just run the always-inline pass and,
427 // if enabled, the function merging pass.
429 addPGOInstrPasses(MPM);
435 // FIXME: The BarrierNoopPass is a HACK! The inliner pass above implicitly
436 // creates a CGSCC pass manager, but we don't want to add extensions into
437 // that pass manager. To prevent this we insert a no-op module pass to reset
438 // the pass manager to get the same behavior as EP_OptimizerLast in non-O0
439 // builds. The function merging pass is
441 MPM.add(createMergeFunctionsPass());
442 else if (!GlobalExtensions->empty() || !Extensions.empty())
443 MPM.add(createBarrierNoopPass());
445 addExtensionsToPM(EP_EnabledOnOptLevel0, MPM);
447 // Rename anon globals to be able to export them in the summary.
448 // This has to be done after we add the extensions to the pass manager
449 // as there could be passes (e.g. Adddress sanitizer) which introduce
450 // new unnamed globals.
451 if (PrepareForThinLTO)
452 MPM.add(createNameAnonGlobalPass());
456 // Add LibraryInfo if we have some.
458 MPM.add(new TargetLibraryInfoWrapperPass(*LibraryInfo));
460 addInitialAliasAnalysisPasses(MPM);
462 // For ThinLTO there are two passes of indirect call promotion. The
463 // first is during the compile phase when PerformThinLTO=false and
464 // intra-module indirect call targets are promoted. The second is during
465 // the ThinLTO backend when PerformThinLTO=true, when we promote imported
466 // inter-module indirect calls. For that we perform indirect call promotion
467 // earlier in the pass pipeline, here before globalopt. Otherwise imported
468 // available_externally functions look unreferenced and are removed.
470 MPM.add(createPGOIndirectCallPromotionLegacyPass(/*InLTO = */ true,
471 !PGOSampleUse.empty()));
473 // For SamplePGO in ThinLTO compile phase, we do not want to unroll loops
474 // as it will change the CFG too much to make the 2nd profile annotation
475 // in backend more difficult.
476 bool PrepareForThinLTOUsingPGOSampleProfile =
477 PrepareForThinLTO && !PGOSampleUse.empty();
478 if (PrepareForThinLTOUsingPGOSampleProfile)
479 DisableUnrollLoops = true;
481 if (!DisableUnitAtATime) {
482 // Infer attributes about declarations if possible.
483 MPM.add(createInferFunctionAttrsLegacyPass());
485 addExtensionsToPM(EP_ModuleOptimizerEarly, MPM);
487 MPM.add(createIPSCCPPass()); // IP SCCP
488 MPM.add(createGlobalOptimizerPass()); // Optimize out global vars
489 // Promote any localized global vars.
490 MPM.add(createPromoteMemoryToRegisterPass());
492 MPM.add(createDeadArgEliminationPass()); // Dead argument elimination
494 addInstructionCombiningPass(MPM); // Clean up after IPCP & DAE
495 addExtensionsToPM(EP_Peephole, MPM);
496 MPM.add(createCFGSimplificationPass()); // Clean up after IPCP & DAE
499 // For SamplePGO in ThinLTO compile phase, we do not want to do indirect
500 // call promotion as it will change the CFG too much to make the 2nd
501 // profile annotation in backend more difficult.
502 // PGO instrumentation is added during the compile phase for ThinLTO, do
503 // not run it a second time
504 if (!PerformThinLTO && !PrepareForThinLTOUsingPGOSampleProfile)
505 addPGOInstrPasses(MPM);
507 if (EnableNonLTOGlobalsModRef)
508 // We add a module alias analysis pass here. In part due to bugs in the
509 // analysis infrastructure this "works" in that the analysis stays alive
510 // for the entire SCC pass run below.
511 MPM.add(createGlobalsAAWrapperPass());
513 // Start of CallGraph SCC passes.
514 if (!DisableUnitAtATime)
515 MPM.add(createPruneEHPass()); // Remove dead EH info
520 if (!DisableUnitAtATime)
521 MPM.add(createPostOrderFunctionAttrsLegacyPass());
523 MPM.add(createArgumentPromotionPass()); // Scalarize uninlined fn args
525 addExtensionsToPM(EP_CGSCCOptimizerLate, MPM);
526 addFunctionSimplificationPasses(MPM);
528 // FIXME: This is a HACK! The inliner pass above implicitly creates a CGSCC
529 // pass manager that we are specifically trying to avoid. To prevent this
530 // we must insert a no-op module pass to reset the pass manager.
531 MPM.add(createBarrierNoopPass());
532 if (RunPartialInlining)
533 MPM.add(createPartialInliningPass());
535 if (!DisableUnitAtATime && OptLevel > 1 && !PrepareForLTO &&
537 // Remove avail extern fns and globals definitions if we aren't
538 // compiling an object file for later LTO. For LTO we want to preserve
539 // these so they are eligible for inlining at link-time. Note if they
540 // are unreferenced they will be removed by GlobalDCE later, so
541 // this only impacts referenced available externally globals.
542 // Eventually they will be suppressed during codegen, but eliminating
543 // here enables more opportunity for GlobalDCE as it may make
544 // globals referenced by available external functions dead
545 // and saves running remaining passes on the eliminated functions.
546 MPM.add(createEliminateAvailableExternallyPass());
548 if (!DisableUnitAtATime)
549 MPM.add(createReversePostOrderFunctionAttrsPass());
551 // If we are planning to perform ThinLTO later, let's not bloat the code with
552 // unrolling/vectorization/... now. We'll first run the inliner + CGSCC passes
553 // during ThinLTO and perform the rest of the optimizations afterward.
554 if (PrepareForThinLTO) {
555 // Reduce the size of the IR as much as possible.
556 MPM.add(createGlobalOptimizerPass());
557 // Rename anon globals to be able to export them in the summary.
558 MPM.add(createNameAnonGlobalPass());
563 // Optimize globals now when performing ThinLTO, this enables more
564 // optimizations later.
565 MPM.add(createGlobalOptimizerPass());
567 // Scheduling LoopVersioningLICM when inlining is over, because after that
568 // we may see more accurate aliasing. Reason to run this late is that too
569 // early versioning may prevent further inlining due to increase of code
570 // size. By placing it just after inlining other optimizations which runs
571 // later might get benefit of no-alias assumption in clone loop.
572 if (UseLoopVersioningLICM) {
573 MPM.add(createLoopVersioningLICMPass()); // Do LoopVersioningLICM
574 MPM.add(createLICMPass()); // Hoist loop invariants
577 if (EnableNonLTOGlobalsModRef)
578 // We add a fresh GlobalsModRef run at this point. This is particularly
579 // useful as the above will have inlined, DCE'ed, and function-attr
580 // propagated everything. We should at this point have a reasonably minimal
581 // and richly annotated call graph. By computing aliasing and mod/ref
582 // information for all local globals here, the late loop passes and notably
583 // the vectorizer will be able to use them to help recognize vectorizable
584 // memory operations.
586 // Note that this relies on a bug in the pass manager which preserves
587 // a module analysis into a function pass pipeline (and throughout it) so
588 // long as the first function pass doesn't invalidate the module analysis.
589 // Thus both Float2Int and LoopRotate have to preserve AliasAnalysis for
590 // this to work. Fortunately, it is trivial to preserve AliasAnalysis
591 // (doing nothing preserves it as it is required to be conservatively
592 // correct in the face of IR changes).
593 MPM.add(createGlobalsAAWrapperPass());
595 MPM.add(createFloat2IntPass());
597 addExtensionsToPM(EP_VectorizerStart, MPM);
599 // Re-rotate loops in all our loop nests. These may have fallout out of
600 // rotated form due to GVN or other transformations, and the vectorizer relies
601 // on the rotated form. Disable header duplication at -Oz.
602 MPM.add(createLoopRotatePass(SizeLevel == 2 ? 0 : -1));
604 // Distribute loops to allow partial vectorization. I.e. isolate dependences
605 // into separate loop that would otherwise inhibit vectorization. This is
606 // currently only performed for loops marked with the metadata
607 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
608 MPM.add(createLoopDistributePass());
610 MPM.add(createLoopVectorizePass(DisableUnrollLoops, LoopVectorize));
612 // Eliminate loads by forwarding stores from the previous iteration to loads
613 // of the current iteration.
614 if (EnableLoopLoadElim)
615 MPM.add(createLoopLoadEliminationPass());
617 // FIXME: Because of #pragma vectorize enable, the passes below are always
618 // inserted in the pipeline, even when the vectorizer doesn't run (ex. when
619 // on -O1 and no #pragma is found). Would be good to have these two passes
620 // as function calls, so that we can only pass them when the vectorizer
622 addInstructionCombiningPass(MPM);
623 if (OptLevel > 1 && ExtraVectorizerPasses) {
624 // At higher optimization levels, try to clean up any runtime overlap and
625 // alignment checks inserted by the vectorizer. We want to track correllated
626 // runtime checks for two inner loops in the same outer loop, fold any
627 // common computations, hoist loop-invariant aspects out of any outer loop,
628 // and unswitch the runtime checks if possible. Once hoisted, we may have
629 // dead (or speculatable) control flows or more combining opportunities.
630 MPM.add(createEarlyCSEPass());
631 MPM.add(createCorrelatedValuePropagationPass());
632 addInstructionCombiningPass(MPM);
633 MPM.add(createLICMPass());
634 MPM.add(createLoopUnswitchPass(SizeLevel || OptLevel < 3, DivergentTarget));
635 MPM.add(createCFGSimplificationPass());
636 addInstructionCombiningPass(MPM);
639 if (RunSLPAfterLoopVectorization) {
641 MPM.add(createSLPVectorizerPass()); // Vectorize parallel scalar chains.
642 if (OptLevel > 1 && ExtraVectorizerPasses) {
643 MPM.add(createEarlyCSEPass());
648 MPM.add(createBBVectorizePass());
649 addInstructionCombiningPass(MPM);
650 addExtensionsToPM(EP_Peephole, MPM);
651 if (OptLevel > 1 && UseGVNAfterVectorization)
654 : createGVNPass(DisableGVNLoadPRE)); // Remove redundancies
656 MPM.add(createEarlyCSEPass()); // Catch trivial redundancies
658 // BBVectorize may have significantly shortened a loop body; unroll again.
659 if (!DisableUnrollLoops)
660 MPM.add(createLoopUnrollPass(OptLevel));
664 addExtensionsToPM(EP_Peephole, MPM);
665 MPM.add(createLateCFGSimplificationPass()); // Switches to lookup tables
666 addInstructionCombiningPass(MPM);
668 if (!DisableUnrollLoops) {
669 MPM.add(createLoopUnrollPass(OptLevel)); // Unroll small loops
671 // LoopUnroll may generate some redundency to cleanup.
672 addInstructionCombiningPass(MPM);
674 // Runtime unrolling will introduce runtime check in loop prologue. If the
675 // unrolled loop is a inner loop, then the prologue will be inside the
676 // outer loop. LICM pass can help to promote the runtime check out if the
677 // checked value is loop invariant.
678 MPM.add(createLICMPass());
681 // After vectorization and unrolling, assume intrinsics may tell us more
682 // about pointer alignments.
683 MPM.add(createAlignmentFromAssumptionsPass());
685 if (!DisableUnitAtATime) {
686 // FIXME: We shouldn't bother with this anymore.
687 MPM.add(createStripDeadPrototypesPass()); // Get rid of dead prototypes
689 // GlobalOpt already deletes dead functions and globals, at -O2 try a
690 // late pass of GlobalDCE. It is capable of deleting dead cycles.
692 MPM.add(createGlobalDCEPass()); // Remove dead fns and globals.
693 MPM.add(createConstantMergePass()); // Merge dup global constants
698 MPM.add(createMergeFunctionsPass());
700 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
701 // canonicalization pass that enables other optimizations. As a result,
702 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
704 MPM.add(createLoopSinkPass());
705 // Get rid of LCSSA nodes.
706 MPM.add(createInstructionSimplifierPass());
708 // LoopSink (and other loop passes since the last simplifyCFG) might have
709 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
710 MPM.add(createCFGSimplificationPass());
712 addExtensionsToPM(EP_OptimizerLast, MPM);
715 void PassManagerBuilder::addLTOOptimizationPasses(legacy::PassManagerBase &PM) {
716 // Remove unused virtual tables to improve the quality of code generated by
717 // whole-program devirtualization and bitset lowering.
718 PM.add(createGlobalDCEPass());
720 // Provide AliasAnalysis services for optimizations.
721 addInitialAliasAnalysisPasses(PM);
723 // Allow forcing function attributes as a debugging and tuning aid.
724 PM.add(createForceFunctionAttrsLegacyPass());
726 // Infer attributes about declarations if possible.
727 PM.add(createInferFunctionAttrsLegacyPass());
730 // Indirect call promotion. This should promote all the targets that are
731 // left by the earlier promotion pass that promotes intra-module targets.
732 // This two-step promotion is to save the compile time. For LTO, it should
733 // produce the same result as if we only do promotion here.
735 createPGOIndirectCallPromotionLegacyPass(true, !PGOSampleUse.empty()));
737 // Propagate constants at call sites into the functions they call. This
738 // opens opportunities for globalopt (and inlining) by substituting function
739 // pointers passed as arguments to direct uses of functions.
740 PM.add(createIPSCCPPass());
743 // Infer attributes about definitions. The readnone attribute in particular is
744 // required for virtual constant propagation.
745 PM.add(createPostOrderFunctionAttrsLegacyPass());
746 PM.add(createReversePostOrderFunctionAttrsPass());
748 // Split globals using inrange annotations on GEP indices. This can help
749 // improve the quality of generated code when virtual constant propagation or
750 // control flow integrity are enabled.
751 PM.add(createGlobalSplitPass());
753 // Apply whole-program devirtualization and virtual constant propagation.
754 PM.add(createWholeProgramDevirtPass(ExportSummary, nullptr));
756 // That's all we need at opt level 1.
760 // Now that we internalized some globals, see if we can hack on them!
761 PM.add(createGlobalOptimizerPass());
762 // Promote any localized global vars.
763 PM.add(createPromoteMemoryToRegisterPass());
765 // Linking modules together can lead to duplicated global constants, only
766 // keep one copy of each constant.
767 PM.add(createConstantMergePass());
769 // Remove unused arguments from functions.
770 PM.add(createDeadArgEliminationPass());
772 // Reduce the code after globalopt and ipsccp. Both can open up significant
773 // simplification opportunities, and both can propagate functions through
774 // function pointers. When this happens, we often have to resolve varargs
775 // calls, etc, so let instcombine do this.
776 addInstructionCombiningPass(PM);
777 addExtensionsToPM(EP_Peephole, PM);
779 // Inline small functions
780 bool RunInliner = Inliner;
786 PM.add(createPruneEHPass()); // Remove dead EH info.
788 // Optimize globals again if we ran the inliner.
790 PM.add(createGlobalOptimizerPass());
791 PM.add(createGlobalDCEPass()); // Remove dead functions.
793 // If we didn't decide to inline a function, check to see if we can
794 // transform it to pass arguments by value instead of by reference.
795 PM.add(createArgumentPromotionPass());
797 // The IPO passes may leave cruft around. Clean up after them.
798 addInstructionCombiningPass(PM);
799 addExtensionsToPM(EP_Peephole, PM);
800 PM.add(createJumpThreadingPass());
803 PM.add(createSROAPass());
805 // Run a few AA driven optimizations here and now, to cleanup the code.
806 PM.add(createPostOrderFunctionAttrsLegacyPass()); // Add nocapture.
807 PM.add(createGlobalsAAWrapperPass()); // IP alias analysis.
809 PM.add(createLICMPass()); // Hoist loop invariants.
810 PM.add(createMergedLoadStoreMotionPass()); // Merge ld/st in diamonds.
811 PM.add(NewGVN ? createNewGVNPass()
812 : createGVNPass(DisableGVNLoadPRE)); // Remove redundancies.
813 PM.add(createMemCpyOptPass()); // Remove dead memcpys.
816 PM.add(createDeadStoreEliminationPass());
818 // More loops are countable; try to optimize them.
819 PM.add(createIndVarSimplifyPass());
820 PM.add(createLoopDeletionPass());
821 if (EnableLoopInterchange)
822 PM.add(createLoopInterchangePass());
824 if (!DisableUnrollLoops)
825 PM.add(createSimpleLoopUnrollPass(OptLevel)); // Unroll small loops
826 PM.add(createLoopVectorizePass(true, LoopVectorize));
827 // The vectorizer may have significantly shortened a loop body; unroll again.
828 if (!DisableUnrollLoops)
829 PM.add(createLoopUnrollPass(OptLevel));
831 // Now that we've optimized loops (in particular loop induction variables),
832 // we may have exposed more scalar opportunities. Run parts of the scalar
833 // optimizer again at this point.
834 addInstructionCombiningPass(PM); // Initial cleanup
835 PM.add(createCFGSimplificationPass()); // if-convert
836 PM.add(createSCCPPass()); // Propagate exposed constants
837 addInstructionCombiningPass(PM); // Clean up again
838 PM.add(createBitTrackingDCEPass());
840 // More scalar chains could be vectorized due to more alias information
841 if (RunSLPAfterLoopVectorization)
843 PM.add(createSLPVectorizerPass()); // Vectorize parallel scalar chains.
845 // After vectorization, assume intrinsics may tell us more about pointer
847 PM.add(createAlignmentFromAssumptionsPass());
850 PM.add(createLoadCombinePass());
852 // Cleanup and simplify the code after the scalar optimizations.
853 addInstructionCombiningPass(PM);
854 addExtensionsToPM(EP_Peephole, PM);
856 PM.add(createJumpThreadingPass());
859 void PassManagerBuilder::addLateLTOOptimizationPasses(
860 legacy::PassManagerBase &PM) {
861 // Delete basic blocks, which optimization passes may have killed.
862 PM.add(createCFGSimplificationPass());
864 // Drop bodies of available externally objects to improve GlobalDCE.
865 PM.add(createEliminateAvailableExternallyPass());
867 // Now that we have optimized the program, discard unreachable functions.
868 PM.add(createGlobalDCEPass());
870 // FIXME: this is profitable (for compiler time) to do at -O0 too, but
871 // currently it damages debug info.
873 PM.add(createMergeFunctionsPass());
876 void PassManagerBuilder::populateThinLTOPassManager(
877 legacy::PassManagerBase &PM) {
878 PerformThinLTO = true;
881 PM.add(createVerifierPass());
884 // These passes import type identifier resolutions for whole-program
885 // devirtualization and CFI. They must run early because other passes may
886 // disturb the specific instruction patterns that these passes look for,
887 // creating dependencies on resolutions that may not appear in the summary.
889 // For example, GVN may transform the pattern assume(type.test) appearing in
890 // two basic blocks into assume(phi(type.test, type.test)), which would
891 // transform a dependency on a WPD resolution into a dependency on a type
892 // identifier resolution for CFI.
894 // Also, WPD has access to more precise information than ICP and can
895 // devirtualize more effectively, so it should operate on the IR first.
896 PM.add(createWholeProgramDevirtPass(nullptr, ImportSummary));
897 PM.add(createLowerTypeTestsPass(nullptr, ImportSummary));
900 populateModulePassManager(PM);
903 PM.add(createVerifierPass());
904 PerformThinLTO = false;
907 void PassManagerBuilder::populateLTOPassManager(legacy::PassManagerBase &PM) {
909 PM.add(new TargetLibraryInfoWrapperPass(*LibraryInfo));
912 PM.add(createVerifierPass());
915 addLTOOptimizationPasses(PM);
917 // The whole-program-devirt pass needs to run at -O0 because only it knows
918 // about the llvm.type.checked.load intrinsic: it needs to both lower the
919 // intrinsic itself and handle it in the summary.
920 PM.add(createWholeProgramDevirtPass(ExportSummary, nullptr));
923 // Create a function that performs CFI checks for cross-DSO calls with targets
924 // in the current module.
925 PM.add(createCrossDSOCFIPass());
927 // Lower type metadata and the type.test intrinsic. This pass supports Clang's
928 // control flow integrity mechanisms (-fsanitize=cfi*) and needs to run at
929 // link time if CFI is enabled. The pass does nothing if CFI is disabled.
930 PM.add(createLowerTypeTestsPass(ExportSummary, nullptr));
933 addLateLTOOptimizationPasses(PM);
936 PM.add(createVerifierPass());
939 inline PassManagerBuilder *unwrap(LLVMPassManagerBuilderRef P) {
940 return reinterpret_cast<PassManagerBuilder*>(P);
943 inline LLVMPassManagerBuilderRef wrap(PassManagerBuilder *P) {
944 return reinterpret_cast<LLVMPassManagerBuilderRef>(P);
947 LLVMPassManagerBuilderRef LLVMPassManagerBuilderCreate() {
948 PassManagerBuilder *PMB = new PassManagerBuilder();
952 void LLVMPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) {
953 PassManagerBuilder *Builder = unwrap(PMB);
958 LLVMPassManagerBuilderSetOptLevel(LLVMPassManagerBuilderRef PMB,
960 PassManagerBuilder *Builder = unwrap(PMB);
961 Builder->OptLevel = OptLevel;
965 LLVMPassManagerBuilderSetSizeLevel(LLVMPassManagerBuilderRef PMB,
966 unsigned SizeLevel) {
967 PassManagerBuilder *Builder = unwrap(PMB);
968 Builder->SizeLevel = SizeLevel;
972 LLVMPassManagerBuilderSetDisableUnitAtATime(LLVMPassManagerBuilderRef PMB,
974 PassManagerBuilder *Builder = unwrap(PMB);
975 Builder->DisableUnitAtATime = Value;
979 LLVMPassManagerBuilderSetDisableUnrollLoops(LLVMPassManagerBuilderRef PMB,
981 PassManagerBuilder *Builder = unwrap(PMB);
982 Builder->DisableUnrollLoops = Value;
986 LLVMPassManagerBuilderSetDisableSimplifyLibCalls(LLVMPassManagerBuilderRef PMB,
988 // NOTE: The simplify-libcalls pass has been removed.
992 LLVMPassManagerBuilderUseInlinerWithThreshold(LLVMPassManagerBuilderRef PMB,
993 unsigned Threshold) {
994 PassManagerBuilder *Builder = unwrap(PMB);
995 Builder->Inliner = createFunctionInliningPass(Threshold);
999 LLVMPassManagerBuilderPopulateFunctionPassManager(LLVMPassManagerBuilderRef PMB,
1000 LLVMPassManagerRef PM) {
1001 PassManagerBuilder *Builder = unwrap(PMB);
1002 legacy::FunctionPassManager *FPM = unwrap<legacy::FunctionPassManager>(PM);
1003 Builder->populateFunctionPassManager(*FPM);
1007 LLVMPassManagerBuilderPopulateModulePassManager(LLVMPassManagerBuilderRef PMB,
1008 LLVMPassManagerRef PM) {
1009 PassManagerBuilder *Builder = unwrap(PMB);
1010 legacy::PassManagerBase *MPM = unwrap(PM);
1011 Builder->populateModulePassManager(*MPM);
1014 void LLVMPassManagerBuilderPopulateLTOPassManager(LLVMPassManagerBuilderRef PMB,
1015 LLVMPassManagerRef PM,
1016 LLVMBool Internalize,
1017 LLVMBool RunInliner) {
1018 PassManagerBuilder *Builder = unwrap(PMB);
1019 legacy::PassManagerBase *LPM = unwrap(PM);
1021 // A small backwards compatibility hack. populateLTOPassManager used to take
1022 // an RunInliner option.
1023 if (RunInliner && !Builder->Inliner)
1024 Builder->Inliner = createFunctionInliningPass();
1026 Builder->populateLTOPassManager(*LPM);