1 //===- PassManagerBuilder.cpp - Build Standard Pass -----------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the PassManagerBuilder class, which is used to set up a
11 // "standard" optimization sequence suitable for languages like C and C++.
13 //===----------------------------------------------------------------------===//
15 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
16 #include "llvm-c/Transforms/PassManagerBuilder.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/Analysis/BasicAliasAnalysis.h"
19 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
20 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
21 #include "llvm/Analysis/GlobalsModRef.h"
22 #include "llvm/Analysis/InlineCost.h"
23 #include "llvm/Analysis/Passes.h"
24 #include "llvm/Analysis/ScopedNoAliasAA.h"
25 #include "llvm/Analysis/TargetLibraryInfo.h"
26 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
27 #include "llvm/IR/DataLayout.h"
28 #include "llvm/IR/LegacyPassManager.h"
29 #include "llvm/IR/ModuleSummaryIndex.h"
30 #include "llvm/IR/Verifier.h"
31 #include "llvm/Support/CommandLine.h"
32 #include "llvm/Support/ManagedStatic.h"
33 #include "llvm/Target/TargetMachine.h"
34 #include "llvm/Transforms/IPO.h"
35 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
36 #include "llvm/Transforms/IPO/FunctionAttrs.h"
37 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
38 #include "llvm/Transforms/Instrumentation.h"
39 #include "llvm/Transforms/Scalar.h"
40 #include "llvm/Transforms/Scalar/GVN.h"
41 #include "llvm/Transforms/Vectorize.h"
46 RunLoopVectorization("vectorize-loops", cl::Hidden,
47 cl::desc("Run the Loop vectorization passes"));
50 RunSLPVectorization("vectorize-slp", cl::Hidden,
51 cl::desc("Run the SLP vectorization passes"));
54 RunBBVectorization("vectorize-slp-aggressive", cl::Hidden,
55 cl::desc("Run the BB vectorization passes"));
58 UseGVNAfterVectorization("use-gvn-after-vectorization",
59 cl::init(false), cl::Hidden,
60 cl::desc("Run GVN instead of Early CSE after vectorization passes"));
62 static cl::opt<bool> ExtraVectorizerPasses(
63 "extra-vectorizer-passes", cl::init(false), cl::Hidden,
64 cl::desc("Run cleanup optimization passes after vectorization."));
67 RunLoopRerolling("reroll-loops", cl::Hidden,
68 cl::desc("Run the loop rerolling pass"));
70 static cl::opt<bool> RunLoadCombine("combine-loads", cl::init(false),
72 cl::desc("Run the load combining pass"));
74 static cl::opt<bool> RunNewGVN("enable-newgvn", cl::init(false), cl::Hidden,
75 cl::desc("Run the NewGVN pass"));
78 RunSLPAfterLoopVectorization("run-slp-after-loop-vectorization",
79 cl::init(true), cl::Hidden,
80 cl::desc("Run the SLP vectorizer (and BB vectorizer) after the Loop "
81 "vectorizer instead of before"));
83 // Experimental option to use CFL-AA
84 enum class CFLAAType { None, Steensgaard, Andersen, Both };
85 static cl::opt<CFLAAType>
86 UseCFLAA("use-cfl-aa", cl::init(CFLAAType::None), cl::Hidden,
87 cl::desc("Enable the new, experimental CFL alias analysis"),
88 cl::values(clEnumValN(CFLAAType::None, "none", "Disable CFL-AA"),
89 clEnumValN(CFLAAType::Steensgaard, "steens",
90 "Enable unification-based CFL-AA"),
91 clEnumValN(CFLAAType::Andersen, "anders",
92 "Enable inclusion-based CFL-AA"),
93 clEnumValN(CFLAAType::Both, "both",
94 "Enable both variants of CFL-AA")));
96 static cl::opt<bool> EnableLoopInterchange(
97 "enable-loopinterchange", cl::init(false), cl::Hidden,
98 cl::desc("Enable the new, experimental LoopInterchange Pass"));
100 static cl::opt<bool> EnableNonLTOGlobalsModRef(
101 "enable-non-lto-gmr", cl::init(true), cl::Hidden,
103 "Enable the GlobalsModRef AliasAnalysis outside of the LTO pipeline."));
105 static cl::opt<bool> EnableLoopLoadElim(
106 "enable-loop-load-elim", cl::init(true), cl::Hidden,
107 cl::desc("Enable the LoopLoadElimination Pass"));
110 EnablePrepareForThinLTO("prepare-for-thinlto", cl::init(false), cl::Hidden,
111 cl::desc("Enable preparation for ThinLTO."));
113 static cl::opt<bool> RunPGOInstrGen(
114 "profile-generate", cl::init(false), cl::Hidden,
115 cl::desc("Enable PGO instrumentation."));
117 static cl::opt<std::string>
118 PGOOutputFile("profile-generate-file", cl::init(""), cl::Hidden,
119 cl::desc("Specify the path of profile data file."));
121 static cl::opt<std::string> RunPGOInstrUse(
122 "profile-use", cl::init(""), cl::Hidden, cl::value_desc("filename"),
123 cl::desc("Enable use phase of PGO instrumentation and specify the path "
124 "of profile data file"));
126 static cl::opt<bool> UseLoopVersioningLICM(
127 "enable-loop-versioning-licm", cl::init(false), cl::Hidden,
128 cl::desc("Enable the experimental Loop Versioning LICM pass"));
131 DisablePreInliner("disable-preinline", cl::init(false), cl::Hidden,
132 cl::desc("Disable pre-instrumentation inliner"));
134 static cl::opt<int> PreInlineThreshold(
135 "preinline-threshold", cl::Hidden, cl::init(75), cl::ZeroOrMore,
136 cl::desc("Control the amount of inlining in pre-instrumentation inliner "
139 static cl::opt<bool> EnableGVNHoist(
140 "enable-gvn-hoist", cl::init(true), cl::Hidden,
141 cl::desc("Enable the GVN hoisting pass (default = on)"));
144 DisableLibCallsShrinkWrap("disable-libcalls-shrinkwrap", cl::init(false),
146 cl::desc("Disable shrink-wrap library calls"));
148 PassManagerBuilder::PassManagerBuilder() {
151 LibraryInfo = nullptr;
153 DisableUnitAtATime = false;
154 DisableUnrollLoops = false;
155 BBVectorize = RunBBVectorization;
156 SLPVectorize = RunSLPVectorization;
157 LoopVectorize = RunLoopVectorization;
158 RerollLoops = RunLoopRerolling;
159 LoadCombine = RunLoadCombine;
161 DisableGVNLoadPRE = false;
163 VerifyOutput = false;
164 MergeFunctions = false;
165 PrepareForLTO = false;
166 EnablePGOInstrGen = RunPGOInstrGen;
167 PGOInstrGen = PGOOutputFile;
168 PGOInstrUse = RunPGOInstrUse;
169 PrepareForThinLTO = EnablePrepareForThinLTO;
170 PerformThinLTO = false;
171 DivergentTarget = false;
174 PassManagerBuilder::~PassManagerBuilder() {
179 /// Set of global extensions, automatically added as part of the standard set.
180 static ManagedStatic<SmallVector<std::pair<PassManagerBuilder::ExtensionPointTy,
181 PassManagerBuilder::ExtensionFn>, 8> > GlobalExtensions;
183 void PassManagerBuilder::addGlobalExtension(
184 PassManagerBuilder::ExtensionPointTy Ty,
185 PassManagerBuilder::ExtensionFn Fn) {
186 GlobalExtensions->push_back(std::make_pair(Ty, std::move(Fn)));
189 void PassManagerBuilder::addExtension(ExtensionPointTy Ty, ExtensionFn Fn) {
190 Extensions.push_back(std::make_pair(Ty, std::move(Fn)));
193 void PassManagerBuilder::addExtensionsToPM(ExtensionPointTy ETy,
194 legacy::PassManagerBase &PM) const {
195 for (unsigned i = 0, e = GlobalExtensions->size(); i != e; ++i)
196 if ((*GlobalExtensions)[i].first == ETy)
197 (*GlobalExtensions)[i].second(*this, PM);
198 for (unsigned i = 0, e = Extensions.size(); i != e; ++i)
199 if (Extensions[i].first == ETy)
200 Extensions[i].second(*this, PM);
203 void PassManagerBuilder::addInitialAliasAnalysisPasses(
204 legacy::PassManagerBase &PM) const {
206 case CFLAAType::Steensgaard:
207 PM.add(createCFLSteensAAWrapperPass());
209 case CFLAAType::Andersen:
210 PM.add(createCFLAndersAAWrapperPass());
212 case CFLAAType::Both:
213 PM.add(createCFLSteensAAWrapperPass());
214 PM.add(createCFLAndersAAWrapperPass());
220 // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
221 // BasicAliasAnalysis wins if they disagree. This is intended to help
222 // support "obvious" type-punning idioms.
223 PM.add(createTypeBasedAAWrapperPass());
224 PM.add(createScopedNoAliasAAWrapperPass());
227 void PassManagerBuilder::addInstructionCombiningPass(
228 legacy::PassManagerBase &PM) const {
229 bool ExpensiveCombines = OptLevel > 2;
230 PM.add(createInstructionCombiningPass(ExpensiveCombines));
233 void PassManagerBuilder::populateFunctionPassManager(
234 legacy::FunctionPassManager &FPM) {
235 addExtensionsToPM(EP_EarlyAsPossible, FPM);
237 // Add LibraryInfo if we have some.
239 FPM.add(new TargetLibraryInfoWrapperPass(*LibraryInfo));
241 if (OptLevel == 0) return;
243 addInitialAliasAnalysisPasses(FPM);
245 FPM.add(createCFGSimplificationPass());
246 FPM.add(createSROAPass());
247 FPM.add(createEarlyCSEPass());
248 FPM.add(createLowerExpectIntrinsicPass());
251 // Do PGO instrumentation generation or use pass as the option specified.
252 void PassManagerBuilder::addPGOInstrPasses(legacy::PassManagerBase &MPM) {
253 if (!EnablePGOInstrGen && PGOInstrUse.empty())
255 // Perform the preinline and cleanup passes for O1 and above.
256 // And avoid doing them if optimizing for size.
257 if (OptLevel > 0 && SizeLevel == 0 && !DisablePreInliner) {
258 // Create preinline pass. We construct an InlineParams object and specify
259 // the threshold here to avoid the command line options of the regular
260 // inliner to influence pre-inlining. The only fields of InlineParams we
261 // care about are DefaultThreshold and HintThreshold.
263 IP.DefaultThreshold = PreInlineThreshold;
264 // FIXME: The hint threshold has the same value used by the regular inliner.
265 // This should probably be lowered after performance testing.
266 IP.HintThreshold = 325;
268 MPM.add(createFunctionInliningPass(IP));
269 MPM.add(createSROAPass());
270 MPM.add(createEarlyCSEPass()); // Catch trivial redundancies
271 MPM.add(createCFGSimplificationPass()); // Merge & remove BBs
272 MPM.add(createInstructionCombiningPass()); // Combine silly seq's
273 addExtensionsToPM(EP_Peephole, MPM);
275 if (EnablePGOInstrGen) {
276 MPM.add(createPGOInstrumentationGenLegacyPass());
277 // Add the profile lowering pass.
278 InstrProfOptions Options;
279 if (!PGOInstrGen.empty())
280 Options.InstrProfileOutput = PGOInstrGen;
281 MPM.add(createInstrProfilingLegacyPass(Options));
283 if (!PGOInstrUse.empty())
284 MPM.add(createPGOInstrumentationUseLegacyPass(PGOInstrUse));
285 // Indirect call promotion that promotes intra-module targets only.
286 // For ThinLTO this is done earlier due to interactions with globalopt
287 // for imported functions. We don't run this at -O0.
290 createPGOIndirectCallPromotionLegacyPass(false, !PGOSampleUse.empty()));
292 void PassManagerBuilder::addFunctionSimplificationPasses(
293 legacy::PassManagerBase &MPM) {
294 // Start of function pass.
295 // Break up aggregate allocas, using SSAUpdater.
296 MPM.add(createSROAPass());
297 MPM.add(createEarlyCSEPass()); // Catch trivial redundancies
299 MPM.add(createGVNHoistPass());
300 // Speculative execution if the target has divergent branches; otherwise nop.
301 MPM.add(createSpeculativeExecutionIfHasBranchDivergencePass());
302 MPM.add(createJumpThreadingPass()); // Thread jumps.
303 MPM.add(createCorrelatedValuePropagationPass()); // Propagate conditionals
304 MPM.add(createCFGSimplificationPass()); // Merge & remove BBs
305 // Combine silly seq's
306 addInstructionCombiningPass(MPM);
307 if (SizeLevel == 0 && !DisableLibCallsShrinkWrap)
308 MPM.add(createLibCallsShrinkWrapPass());
309 addExtensionsToPM(EP_Peephole, MPM);
311 // Optimize memory intrinsic calls based on the profiled size information.
313 MPM.add(createPGOMemOPSizeOptLegacyPass());
315 MPM.add(createTailCallEliminationPass()); // Eliminate tail calls
316 MPM.add(createCFGSimplificationPass()); // Merge & remove BBs
317 MPM.add(createReassociatePass()); // Reassociate expressions
318 // Rotate Loop - disable header duplication at -Oz
319 MPM.add(createLoopRotatePass(SizeLevel == 2 ? 0 : -1));
320 MPM.add(createLICMPass()); // Hoist loop invariants
321 MPM.add(createLoopUnswitchPass(SizeLevel || OptLevel < 3, DivergentTarget));
322 MPM.add(createCFGSimplificationPass());
323 addInstructionCombiningPass(MPM);
324 MPM.add(createIndVarSimplifyPass()); // Canonicalize indvars
325 MPM.add(createLoopIdiomPass()); // Recognize idioms like memset.
326 addExtensionsToPM(EP_LateLoopOptimizations, MPM);
327 MPM.add(createLoopDeletionPass()); // Delete dead loops
329 if (EnableLoopInterchange) {
330 MPM.add(createLoopInterchangePass()); // Interchange loops
331 MPM.add(createCFGSimplificationPass());
333 if (!DisableUnrollLoops)
334 MPM.add(createSimpleLoopUnrollPass(OptLevel)); // Unroll small loops
335 addExtensionsToPM(EP_LoopOptimizerEnd, MPM);
338 MPM.add(createMergedLoadStoreMotionPass()); // Merge ld/st in diamonds
339 MPM.add(NewGVN ? createNewGVNPass()
340 : createGVNPass(DisableGVNLoadPRE)); // Remove redundancies
342 MPM.add(createMemCpyOptPass()); // Remove memcpy / form memset
343 MPM.add(createSCCPPass()); // Constant prop with SCCP
345 // Delete dead bit computations (instcombine runs after to fold away the dead
346 // computations, and then ADCE will run later to exploit any new DCE
347 // opportunities that creates).
348 MPM.add(createBitTrackingDCEPass()); // Delete dead bit computations
350 // Run instcombine after redundancy elimination to exploit opportunities
351 // opened up by them.
352 addInstructionCombiningPass(MPM);
353 addExtensionsToPM(EP_Peephole, MPM);
354 MPM.add(createJumpThreadingPass()); // Thread jumps
355 MPM.add(createCorrelatedValuePropagationPass());
356 MPM.add(createDeadStoreEliminationPass()); // Delete dead stores
357 MPM.add(createLICMPass());
359 addExtensionsToPM(EP_ScalarOptimizerLate, MPM);
362 MPM.add(createLoopRerollPass());
363 if (!RunSLPAfterLoopVectorization) {
365 MPM.add(createSLPVectorizerPass()); // Vectorize parallel scalar chains.
368 MPM.add(createBBVectorizePass());
369 addInstructionCombiningPass(MPM);
370 addExtensionsToPM(EP_Peephole, MPM);
371 if (OptLevel > 1 && UseGVNAfterVectorization)
374 : createGVNPass(DisableGVNLoadPRE)); // Remove redundancies
376 MPM.add(createEarlyCSEPass()); // Catch trivial redundancies
378 // BBVectorize may have significantly shortened a loop body; unroll again.
379 if (!DisableUnrollLoops)
380 MPM.add(createLoopUnrollPass(OptLevel));
385 MPM.add(createLoadCombinePass());
387 MPM.add(createAggressiveDCEPass()); // Delete dead instructions
388 MPM.add(createCFGSimplificationPass()); // Merge & remove BBs
389 // Clean up after everything.
390 addInstructionCombiningPass(MPM);
391 addExtensionsToPM(EP_Peephole, MPM);
394 void PassManagerBuilder::populateModulePassManager(
395 legacy::PassManagerBase &MPM) {
396 if (!PGOSampleUse.empty()) {
397 MPM.add(createPruneEHPass());
398 MPM.add(createSampleProfileLoaderPass(PGOSampleUse));
401 // Allow forcing function attributes as a debugging and tuning aid.
402 MPM.add(createForceFunctionAttrsLegacyPass());
404 // If all optimizations are disabled, just run the always-inline pass and,
405 // if enabled, the function merging pass.
407 addPGOInstrPasses(MPM);
413 // FIXME: The BarrierNoopPass is a HACK! The inliner pass above implicitly
414 // creates a CGSCC pass manager, but we don't want to add extensions into
415 // that pass manager. To prevent this we insert a no-op module pass to reset
416 // the pass manager to get the same behavior as EP_OptimizerLast in non-O0
417 // builds. The function merging pass is
419 MPM.add(createMergeFunctionsPass());
420 else if (!GlobalExtensions->empty() || !Extensions.empty())
421 MPM.add(createBarrierNoopPass());
423 addExtensionsToPM(EP_EnabledOnOptLevel0, MPM);
425 // Rename anon globals to be able to export them in the summary.
426 // This has to be done after we add the extensions to the pass manager
427 // as there could be passes (e.g. Adddress sanitizer) which introduce
428 // new unnamed globals.
429 if (PrepareForThinLTO)
430 MPM.add(createNameAnonGlobalPass());
434 // Add LibraryInfo if we have some.
436 MPM.add(new TargetLibraryInfoWrapperPass(*LibraryInfo));
438 addInitialAliasAnalysisPasses(MPM);
440 // For ThinLTO there are two passes of indirect call promotion. The
441 // first is during the compile phase when PerformThinLTO=false and
442 // intra-module indirect call targets are promoted. The second is during
443 // the ThinLTO backend when PerformThinLTO=true, when we promote imported
444 // inter-module indirect calls. For that we perform indirect call promotion
445 // earlier in the pass pipeline, here before globalopt. Otherwise imported
446 // available_externally functions look unreferenced and are removed.
448 MPM.add(createPGOIndirectCallPromotionLegacyPass(/*InLTO = */ true,
449 !PGOSampleUse.empty()));
451 // For SamplePGO in ThinLTO compile phase, we do not want to unroll loops
452 // as it will change the CFG too much to make the 2nd profile annotation
453 // in backend more difficult.
454 bool PrepareForThinLTOUsingPGOSampleProfile =
455 PrepareForThinLTO && !PGOSampleUse.empty();
456 if (PrepareForThinLTOUsingPGOSampleProfile)
457 DisableUnrollLoops = true;
459 if (!DisableUnitAtATime) {
460 // Infer attributes about declarations if possible.
461 MPM.add(createInferFunctionAttrsLegacyPass());
463 addExtensionsToPM(EP_ModuleOptimizerEarly, MPM);
465 MPM.add(createIPSCCPPass()); // IP SCCP
466 MPM.add(createGlobalOptimizerPass()); // Optimize out global vars
467 // Promote any localized global vars.
468 MPM.add(createPromoteMemoryToRegisterPass());
470 MPM.add(createDeadArgEliminationPass()); // Dead argument elimination
472 addInstructionCombiningPass(MPM); // Clean up after IPCP & DAE
473 addExtensionsToPM(EP_Peephole, MPM);
474 MPM.add(createCFGSimplificationPass()); // Clean up after IPCP & DAE
477 // For SamplePGO in ThinLTO compile phase, we do not want to do indirect
478 // call promotion as it will change the CFG too much to make the 2nd
479 // profile annotation in backend more difficult.
480 // PGO instrumentation is added during the compile phase for ThinLTO, do
481 // not run it a second time
482 if (!PerformThinLTO && !PrepareForThinLTOUsingPGOSampleProfile)
483 addPGOInstrPasses(MPM);
485 if (EnableNonLTOGlobalsModRef)
486 // We add a module alias analysis pass here. In part due to bugs in the
487 // analysis infrastructure this "works" in that the analysis stays alive
488 // for the entire SCC pass run below.
489 MPM.add(createGlobalsAAWrapperPass());
491 // Start of CallGraph SCC passes.
492 if (!DisableUnitAtATime)
493 MPM.add(createPruneEHPass()); // Remove dead EH info
498 if (!DisableUnitAtATime)
499 MPM.add(createPostOrderFunctionAttrsLegacyPass());
501 MPM.add(createArgumentPromotionPass()); // Scalarize uninlined fn args
503 addExtensionsToPM(EP_CGSCCOptimizerLate, MPM);
504 addFunctionSimplificationPasses(MPM);
506 // FIXME: This is a HACK! The inliner pass above implicitly creates a CGSCC
507 // pass manager that we are specifically trying to avoid. To prevent this
508 // we must insert a no-op module pass to reset the pass manager.
509 MPM.add(createBarrierNoopPass());
511 if (!DisableUnitAtATime && OptLevel > 1 && !PrepareForLTO &&
513 // Remove avail extern fns and globals definitions if we aren't
514 // compiling an object file for later LTO. For LTO we want to preserve
515 // these so they are eligible for inlining at link-time. Note if they
516 // are unreferenced they will be removed by GlobalDCE later, so
517 // this only impacts referenced available externally globals.
518 // Eventually they will be suppressed during codegen, but eliminating
519 // here enables more opportunity for GlobalDCE as it may make
520 // globals referenced by available external functions dead
521 // and saves running remaining passes on the eliminated functions.
522 MPM.add(createEliminateAvailableExternallyPass());
524 if (!DisableUnitAtATime)
525 MPM.add(createReversePostOrderFunctionAttrsPass());
527 // If we are planning to perform ThinLTO later, let's not bloat the code with
528 // unrolling/vectorization/... now. We'll first run the inliner + CGSCC passes
529 // during ThinLTO and perform the rest of the optimizations afterward.
530 if (PrepareForThinLTO) {
531 // Reduce the size of the IR as much as possible.
532 MPM.add(createGlobalOptimizerPass());
533 // Rename anon globals to be able to export them in the summary.
534 MPM.add(createNameAnonGlobalPass());
539 // Optimize globals now when performing ThinLTO, this enables more
540 // optimizations later.
541 MPM.add(createGlobalOptimizerPass());
543 // Scheduling LoopVersioningLICM when inlining is over, because after that
544 // we may see more accurate aliasing. Reason to run this late is that too
545 // early versioning may prevent further inlining due to increase of code
546 // size. By placing it just after inlining other optimizations which runs
547 // later might get benefit of no-alias assumption in clone loop.
548 if (UseLoopVersioningLICM) {
549 MPM.add(createLoopVersioningLICMPass()); // Do LoopVersioningLICM
550 MPM.add(createLICMPass()); // Hoist loop invariants
553 if (EnableNonLTOGlobalsModRef)
554 // We add a fresh GlobalsModRef run at this point. This is particularly
555 // useful as the above will have inlined, DCE'ed, and function-attr
556 // propagated everything. We should at this point have a reasonably minimal
557 // and richly annotated call graph. By computing aliasing and mod/ref
558 // information for all local globals here, the late loop passes and notably
559 // the vectorizer will be able to use them to help recognize vectorizable
560 // memory operations.
562 // Note that this relies on a bug in the pass manager which preserves
563 // a module analysis into a function pass pipeline (and throughout it) so
564 // long as the first function pass doesn't invalidate the module analysis.
565 // Thus both Float2Int and LoopRotate have to preserve AliasAnalysis for
566 // this to work. Fortunately, it is trivial to preserve AliasAnalysis
567 // (doing nothing preserves it as it is required to be conservatively
568 // correct in the face of IR changes).
569 MPM.add(createGlobalsAAWrapperPass());
571 MPM.add(createFloat2IntPass());
573 addExtensionsToPM(EP_VectorizerStart, MPM);
575 // Re-rotate loops in all our loop nests. These may have fallout out of
576 // rotated form due to GVN or other transformations, and the vectorizer relies
577 // on the rotated form. Disable header duplication at -Oz.
578 MPM.add(createLoopRotatePass(SizeLevel == 2 ? 0 : -1));
580 // Distribute loops to allow partial vectorization. I.e. isolate dependences
581 // into separate loop that would otherwise inhibit vectorization. This is
582 // currently only performed for loops marked with the metadata
583 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
584 MPM.add(createLoopDistributePass());
586 MPM.add(createLoopVectorizePass(DisableUnrollLoops, LoopVectorize));
588 // Eliminate loads by forwarding stores from the previous iteration to loads
589 // of the current iteration.
590 if (EnableLoopLoadElim)
591 MPM.add(createLoopLoadEliminationPass());
593 // FIXME: Because of #pragma vectorize enable, the passes below are always
594 // inserted in the pipeline, even when the vectorizer doesn't run (ex. when
595 // on -O1 and no #pragma is found). Would be good to have these two passes
596 // as function calls, so that we can only pass them when the vectorizer
598 addInstructionCombiningPass(MPM);
599 if (OptLevel > 1 && ExtraVectorizerPasses) {
600 // At higher optimization levels, try to clean up any runtime overlap and
601 // alignment checks inserted by the vectorizer. We want to track correllated
602 // runtime checks for two inner loops in the same outer loop, fold any
603 // common computations, hoist loop-invariant aspects out of any outer loop,
604 // and unswitch the runtime checks if possible. Once hoisted, we may have
605 // dead (or speculatable) control flows or more combining opportunities.
606 MPM.add(createEarlyCSEPass());
607 MPM.add(createCorrelatedValuePropagationPass());
608 addInstructionCombiningPass(MPM);
609 MPM.add(createLICMPass());
610 MPM.add(createLoopUnswitchPass(SizeLevel || OptLevel < 3, DivergentTarget));
611 MPM.add(createCFGSimplificationPass());
612 addInstructionCombiningPass(MPM);
615 if (RunSLPAfterLoopVectorization) {
617 MPM.add(createSLPVectorizerPass()); // Vectorize parallel scalar chains.
618 if (OptLevel > 1 && ExtraVectorizerPasses) {
619 MPM.add(createEarlyCSEPass());
624 MPM.add(createBBVectorizePass());
625 addInstructionCombiningPass(MPM);
626 addExtensionsToPM(EP_Peephole, MPM);
627 if (OptLevel > 1 && UseGVNAfterVectorization)
630 : createGVNPass(DisableGVNLoadPRE)); // Remove redundancies
632 MPM.add(createEarlyCSEPass()); // Catch trivial redundancies
634 // BBVectorize may have significantly shortened a loop body; unroll again.
635 if (!DisableUnrollLoops)
636 MPM.add(createLoopUnrollPass(OptLevel));
640 addExtensionsToPM(EP_Peephole, MPM);
641 MPM.add(createLateCFGSimplificationPass()); // Switches to lookup tables
642 addInstructionCombiningPass(MPM);
644 if (!DisableUnrollLoops) {
645 MPM.add(createLoopUnrollPass(OptLevel)); // Unroll small loops
647 // LoopUnroll may generate some redundency to cleanup.
648 addInstructionCombiningPass(MPM);
650 // Runtime unrolling will introduce runtime check in loop prologue. If the
651 // unrolled loop is a inner loop, then the prologue will be inside the
652 // outer loop. LICM pass can help to promote the runtime check out if the
653 // checked value is loop invariant.
654 MPM.add(createLICMPass());
657 // After vectorization and unrolling, assume intrinsics may tell us more
658 // about pointer alignments.
659 MPM.add(createAlignmentFromAssumptionsPass());
661 if (!DisableUnitAtATime) {
662 // FIXME: We shouldn't bother with this anymore.
663 MPM.add(createStripDeadPrototypesPass()); // Get rid of dead prototypes
665 // GlobalOpt already deletes dead functions and globals, at -O2 try a
666 // late pass of GlobalDCE. It is capable of deleting dead cycles.
668 MPM.add(createGlobalDCEPass()); // Remove dead fns and globals.
669 MPM.add(createConstantMergePass()); // Merge dup global constants
674 MPM.add(createMergeFunctionsPass());
676 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
677 // canonicalization pass that enables other optimizations. As a result,
678 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
680 MPM.add(createLoopSinkPass());
681 // Get rid of LCSSA nodes.
682 MPM.add(createInstructionSimplifierPass());
684 // LoopSink (and other loop passes since the last simplifyCFG) might have
685 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
686 MPM.add(createCFGSimplificationPass());
688 addExtensionsToPM(EP_OptimizerLast, MPM);
691 void PassManagerBuilder::addLTOOptimizationPasses(legacy::PassManagerBase &PM) {
692 // Remove unused virtual tables to improve the quality of code generated by
693 // whole-program devirtualization and bitset lowering.
694 PM.add(createGlobalDCEPass());
696 // Provide AliasAnalysis services for optimizations.
697 addInitialAliasAnalysisPasses(PM);
699 // Allow forcing function attributes as a debugging and tuning aid.
700 PM.add(createForceFunctionAttrsLegacyPass());
702 // Infer attributes about declarations if possible.
703 PM.add(createInferFunctionAttrsLegacyPass());
706 // Indirect call promotion. This should promote all the targets that are
707 // left by the earlier promotion pass that promotes intra-module targets.
708 // This two-step promotion is to save the compile time. For LTO, it should
709 // produce the same result as if we only do promotion here.
711 createPGOIndirectCallPromotionLegacyPass(true, !PGOSampleUse.empty()));
713 // Propagate constants at call sites into the functions they call. This
714 // opens opportunities for globalopt (and inlining) by substituting function
715 // pointers passed as arguments to direct uses of functions.
716 PM.add(createIPSCCPPass());
719 // Infer attributes about definitions. The readnone attribute in particular is
720 // required for virtual constant propagation.
721 PM.add(createPostOrderFunctionAttrsLegacyPass());
722 PM.add(createReversePostOrderFunctionAttrsPass());
724 // Split globals using inrange annotations on GEP indices. This can help
725 // improve the quality of generated code when virtual constant propagation or
726 // control flow integrity are enabled.
727 PM.add(createGlobalSplitPass());
729 // Apply whole-program devirtualization and virtual constant propagation.
730 PM.add(createWholeProgramDevirtPass(ExportSummary, nullptr));
732 // That's all we need at opt level 1.
736 // Now that we internalized some globals, see if we can hack on them!
737 PM.add(createGlobalOptimizerPass());
738 // Promote any localized global vars.
739 PM.add(createPromoteMemoryToRegisterPass());
741 // Linking modules together can lead to duplicated global constants, only
742 // keep one copy of each constant.
743 PM.add(createConstantMergePass());
745 // Remove unused arguments from functions.
746 PM.add(createDeadArgEliminationPass());
748 // Reduce the code after globalopt and ipsccp. Both can open up significant
749 // simplification opportunities, and both can propagate functions through
750 // function pointers. When this happens, we often have to resolve varargs
751 // calls, etc, so let instcombine do this.
752 addInstructionCombiningPass(PM);
753 addExtensionsToPM(EP_Peephole, PM);
755 // Inline small functions
756 bool RunInliner = Inliner;
762 PM.add(createPruneEHPass()); // Remove dead EH info.
764 // Optimize globals again if we ran the inliner.
766 PM.add(createGlobalOptimizerPass());
767 PM.add(createGlobalDCEPass()); // Remove dead functions.
769 // If we didn't decide to inline a function, check to see if we can
770 // transform it to pass arguments by value instead of by reference.
771 PM.add(createArgumentPromotionPass());
773 // The IPO passes may leave cruft around. Clean up after them.
774 addInstructionCombiningPass(PM);
775 addExtensionsToPM(EP_Peephole, PM);
776 PM.add(createJumpThreadingPass());
779 PM.add(createSROAPass());
781 // Run a few AA driven optimizations here and now, to cleanup the code.
782 PM.add(createPostOrderFunctionAttrsLegacyPass()); // Add nocapture.
783 PM.add(createGlobalsAAWrapperPass()); // IP alias analysis.
785 PM.add(createLICMPass()); // Hoist loop invariants.
786 PM.add(createMergedLoadStoreMotionPass()); // Merge ld/st in diamonds.
787 PM.add(NewGVN ? createNewGVNPass()
788 : createGVNPass(DisableGVNLoadPRE)); // Remove redundancies.
789 PM.add(createMemCpyOptPass()); // Remove dead memcpys.
792 PM.add(createDeadStoreEliminationPass());
794 // More loops are countable; try to optimize them.
795 PM.add(createIndVarSimplifyPass());
796 PM.add(createLoopDeletionPass());
797 if (EnableLoopInterchange)
798 PM.add(createLoopInterchangePass());
800 if (!DisableUnrollLoops)
801 PM.add(createSimpleLoopUnrollPass(OptLevel)); // Unroll small loops
802 PM.add(createLoopVectorizePass(true, LoopVectorize));
803 // The vectorizer may have significantly shortened a loop body; unroll again.
804 if (!DisableUnrollLoops)
805 PM.add(createLoopUnrollPass(OptLevel));
807 // Now that we've optimized loops (in particular loop induction variables),
808 // we may have exposed more scalar opportunities. Run parts of the scalar
809 // optimizer again at this point.
810 addInstructionCombiningPass(PM); // Initial cleanup
811 PM.add(createCFGSimplificationPass()); // if-convert
812 PM.add(createSCCPPass()); // Propagate exposed constants
813 addInstructionCombiningPass(PM); // Clean up again
814 PM.add(createBitTrackingDCEPass());
816 // More scalar chains could be vectorized due to more alias information
817 if (RunSLPAfterLoopVectorization)
819 PM.add(createSLPVectorizerPass()); // Vectorize parallel scalar chains.
821 // After vectorization, assume intrinsics may tell us more about pointer
823 PM.add(createAlignmentFromAssumptionsPass());
826 PM.add(createLoadCombinePass());
828 // Cleanup and simplify the code after the scalar optimizations.
829 addInstructionCombiningPass(PM);
830 addExtensionsToPM(EP_Peephole, PM);
832 PM.add(createJumpThreadingPass());
835 void PassManagerBuilder::addLateLTOOptimizationPasses(
836 legacy::PassManagerBase &PM) {
837 // Delete basic blocks, which optimization passes may have killed.
838 PM.add(createCFGSimplificationPass());
840 // Drop bodies of available externally objects to improve GlobalDCE.
841 PM.add(createEliminateAvailableExternallyPass());
843 // Now that we have optimized the program, discard unreachable functions.
844 PM.add(createGlobalDCEPass());
846 // FIXME: this is profitable (for compiler time) to do at -O0 too, but
847 // currently it damages debug info.
849 PM.add(createMergeFunctionsPass());
852 void PassManagerBuilder::populateThinLTOPassManager(
853 legacy::PassManagerBase &PM) {
854 PerformThinLTO = true;
857 PM.add(createVerifierPass());
860 // These passes import type identifier resolutions for whole-program
861 // devirtualization and CFI. They must run early because other passes may
862 // disturb the specific instruction patterns that these passes look for,
863 // creating dependencies on resolutions that may not appear in the summary.
865 // For example, GVN may transform the pattern assume(type.test) appearing in
866 // two basic blocks into assume(phi(type.test, type.test)), which would
867 // transform a dependency on a WPD resolution into a dependency on a type
868 // identifier resolution for CFI.
870 // Also, WPD has access to more precise information than ICP and can
871 // devirtualize more effectively, so it should operate on the IR first.
872 PM.add(createWholeProgramDevirtPass(nullptr, ImportSummary));
873 PM.add(createLowerTypeTestsPass(nullptr, ImportSummary));
876 populateModulePassManager(PM);
879 PM.add(createVerifierPass());
880 PerformThinLTO = false;
883 void PassManagerBuilder::populateLTOPassManager(legacy::PassManagerBase &PM) {
885 PM.add(new TargetLibraryInfoWrapperPass(*LibraryInfo));
888 PM.add(createVerifierPass());
891 addLTOOptimizationPasses(PM);
893 // Create a function that performs CFI checks for cross-DSO calls with targets
894 // in the current module.
895 PM.add(createCrossDSOCFIPass());
897 // Lower type metadata and the type.test intrinsic. This pass supports Clang's
898 // control flow integrity mechanisms (-fsanitize=cfi*) and needs to run at
899 // link time if CFI is enabled. The pass does nothing if CFI is disabled.
900 PM.add(createLowerTypeTestsPass(ExportSummary, nullptr));
903 addLateLTOOptimizationPasses(PM);
906 PM.add(createVerifierPass());
909 inline PassManagerBuilder *unwrap(LLVMPassManagerBuilderRef P) {
910 return reinterpret_cast<PassManagerBuilder*>(P);
913 inline LLVMPassManagerBuilderRef wrap(PassManagerBuilder *P) {
914 return reinterpret_cast<LLVMPassManagerBuilderRef>(P);
917 LLVMPassManagerBuilderRef LLVMPassManagerBuilderCreate() {
918 PassManagerBuilder *PMB = new PassManagerBuilder();
922 void LLVMPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) {
923 PassManagerBuilder *Builder = unwrap(PMB);
928 LLVMPassManagerBuilderSetOptLevel(LLVMPassManagerBuilderRef PMB,
930 PassManagerBuilder *Builder = unwrap(PMB);
931 Builder->OptLevel = OptLevel;
935 LLVMPassManagerBuilderSetSizeLevel(LLVMPassManagerBuilderRef PMB,
936 unsigned SizeLevel) {
937 PassManagerBuilder *Builder = unwrap(PMB);
938 Builder->SizeLevel = SizeLevel;
942 LLVMPassManagerBuilderSetDisableUnitAtATime(LLVMPassManagerBuilderRef PMB,
944 PassManagerBuilder *Builder = unwrap(PMB);
945 Builder->DisableUnitAtATime = Value;
949 LLVMPassManagerBuilderSetDisableUnrollLoops(LLVMPassManagerBuilderRef PMB,
951 PassManagerBuilder *Builder = unwrap(PMB);
952 Builder->DisableUnrollLoops = Value;
956 LLVMPassManagerBuilderSetDisableSimplifyLibCalls(LLVMPassManagerBuilderRef PMB,
958 // NOTE: The simplify-libcalls pass has been removed.
962 LLVMPassManagerBuilderUseInlinerWithThreshold(LLVMPassManagerBuilderRef PMB,
963 unsigned Threshold) {
964 PassManagerBuilder *Builder = unwrap(PMB);
965 Builder->Inliner = createFunctionInliningPass(Threshold);
969 LLVMPassManagerBuilderPopulateFunctionPassManager(LLVMPassManagerBuilderRef PMB,
970 LLVMPassManagerRef PM) {
971 PassManagerBuilder *Builder = unwrap(PMB);
972 legacy::FunctionPassManager *FPM = unwrap<legacy::FunctionPassManager>(PM);
973 Builder->populateFunctionPassManager(*FPM);
977 LLVMPassManagerBuilderPopulateModulePassManager(LLVMPassManagerBuilderRef PMB,
978 LLVMPassManagerRef PM) {
979 PassManagerBuilder *Builder = unwrap(PMB);
980 legacy::PassManagerBase *MPM = unwrap(PM);
981 Builder->populateModulePassManager(*MPM);
984 void LLVMPassManagerBuilderPopulateLTOPassManager(LLVMPassManagerBuilderRef PMB,
985 LLVMPassManagerRef PM,
986 LLVMBool Internalize,
987 LLVMBool RunInliner) {
988 PassManagerBuilder *Builder = unwrap(PMB);
989 legacy::PassManagerBase *LPM = unwrap(PM);
991 // A small backwards compatibility hack. populateLTOPassManager used to take
992 // an RunInliner option.
993 if (RunInliner && !Builder->Inliner)
994 Builder->Inliner = createFunctionInliningPass();
996 Builder->populateLTOPassManager(*LPM);