1 //==- ScheduleDAGInstrs.h - MachineInstr Scheduling --------------*- C++ -*-==//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 /// \file Implements the ScheduleDAGInstrs class, which implements scheduling
11 /// for a MachineInstr-based dependency graph.
13 //===----------------------------------------------------------------------===//
15 #ifndef LLVM_CODEGEN_SCHEDULEDAGINSTRS_H
16 #define LLVM_CODEGEN_SCHEDULEDAGINSTRS_H
18 #include "llvm/ADT/MapVector.h"
19 #include "llvm/ADT/SparseMultiSet.h"
20 #include "llvm/ADT/SparseSet.h"
21 #include "llvm/CodeGen/ScheduleDAG.h"
22 #include "llvm/CodeGen/TargetSchedule.h"
23 #include "llvm/Support/Compiler.h"
24 #include "llvm/Target/TargetRegisterInfo.h"
28 class MachineFrameInfo;
29 class MachineLoopInfo;
30 class MachineDominatorTree;
31 class RegPressureTracker;
34 /// An individual mapping from virtual register number to SUnit.
40 VReg2SUnit(unsigned VReg, LaneBitmask LaneMask, SUnit *SU)
41 : VirtReg(VReg), LaneMask(LaneMask), SU(SU) {}
43 unsigned getSparseSetIndex() const {
44 return TargetRegisterInfo::virtReg2Index(VirtReg);
48 /// Mapping from virtual register to SUnit including an operand index.
49 struct VReg2SUnitOperIdx : public VReg2SUnit {
50 unsigned OperandIndex;
52 VReg2SUnitOperIdx(unsigned VReg, LaneBitmask LaneMask,
53 unsigned OperandIndex, SUnit *SU)
54 : VReg2SUnit(VReg, LaneMask, SU), OperandIndex(OperandIndex) {}
57 /// Record a physical register access.
58 /// For non-data-dependent uses, OpIdx == -1.
59 struct PhysRegSUOper {
64 PhysRegSUOper(SUnit *su, int op, unsigned R): SU(su), OpIdx(op), Reg(R) {}
66 unsigned getSparseSetIndex() const { return Reg; }
69 /// Use a SparseMultiSet to track physical registers. Storage is only
70 /// allocated once for the pass. It can be cleared in constant time and reused
71 /// without any frees.
72 typedef SparseMultiSet<PhysRegSUOper, llvm::identity<unsigned>, uint16_t>
75 /// Use SparseSet as a SparseMap by relying on the fact that it never
76 /// compares ValueT's, only unsigned keys. This allows the set to be cleared
77 /// between scheduling regions in constant time as long as ValueT does not
78 /// require a destructor.
79 typedef SparseSet<VReg2SUnit, VirtReg2IndexFunctor> VReg2SUnitMap;
81 /// Track local uses of virtual registers. These uses are gathered by the DAG
82 /// builder and may be consulted by the scheduler to avoid iterating an entire
84 typedef SparseMultiSet<VReg2SUnit, VirtReg2IndexFunctor> VReg2SUnitMultiMap;
86 typedef SparseMultiSet<VReg2SUnitOperIdx, VirtReg2IndexFunctor>
87 VReg2SUnitOperIdxMultiMap;
89 typedef PointerUnion<const Value *, const PseudoSourceValue *> ValueType;
90 struct UnderlyingObject : PointerIntPair<ValueType, 1, bool> {
91 UnderlyingObject(ValueType V, bool MayAlias)
92 : PointerIntPair<ValueType, 1, bool>(V, MayAlias) {}
93 ValueType getValue() const { return getPointer(); }
94 bool mayAlias() const { return getInt(); }
96 typedef SmallVector<UnderlyingObject, 4> UnderlyingObjectsVector;
98 /// A ScheduleDAG for scheduling lists of MachineInstr.
99 class ScheduleDAGInstrs : public ScheduleDAG {
101 const MachineLoopInfo *MLI;
102 const MachineFrameInfo &MFI;
104 /// TargetSchedModel provides an interface to the machine model.
105 TargetSchedModel SchedModel;
107 /// True if the DAG builder should remove kill flags (in preparation for
109 bool RemoveKillFlags;
111 /// The standard DAG builder does not normally include terminators as DAG
112 /// nodes because it does not create the necessary dependencies to prevent
113 /// reordering. A specialized scheduler can override
114 /// TargetInstrInfo::isSchedulingBoundary then enable this flag to indicate
115 /// it has taken responsibility for scheduling the terminator correctly.
116 bool CanHandleTerminators;
118 /// Whether lane masks should get tracked.
121 // State specific to the current scheduling region.
122 // ------------------------------------------------
124 /// The block in which to insert instructions
125 MachineBasicBlock *BB;
127 /// The beginning of the range to be scheduled.
128 MachineBasicBlock::iterator RegionBegin;
130 /// The end of the range to be scheduled.
131 MachineBasicBlock::iterator RegionEnd;
133 /// Instructions in this region (distance(RegionBegin, RegionEnd)).
134 unsigned NumRegionInstrs;
136 /// After calling BuildSchedGraph, each machine instruction in the current
137 /// scheduling region is mapped to an SUnit.
138 DenseMap<MachineInstr*, SUnit*> MISUnitMap;
140 // State internal to DAG building.
141 // -------------------------------
143 /// Defs, Uses - Remember where defs and uses of each register are as we
144 /// iterate upward through the instructions. This is allocated here instead
145 /// of inside BuildSchedGraph to avoid the need for it to be initialized and
146 /// destructed for each block.
150 /// Tracks the last instruction(s) in this region defining each virtual
151 /// register. There may be multiple current definitions for a register with
152 /// disjunct lanemasks.
153 VReg2SUnitMultiMap CurrentVRegDefs;
154 /// Tracks the last instructions in this region using each virtual register.
155 VReg2SUnitOperIdxMultiMap CurrentVRegUses;
157 AliasAnalysis *AAForDep;
159 /// Remember a generic side-effecting instruction as we proceed.
160 /// No other SU ever gets scheduled around it (except in the special
161 /// case of a huge region that gets reduced).
165 /// A list of SUnits, used in Value2SUsMap, during DAG construction.
166 /// Note: to gain speed it might be worth investigating an optimized
167 /// implementation of this data structure, such as a singly linked list
168 /// with a memory pool (SmallVector was tried but slow and SparseSet is not
170 typedef std::list<SUnit *> SUList;
173 /// \brief A map from ValueType to SUList, used during DAG construction, as
174 /// a means of remembering which SUs depend on which memory locations.
177 /// Reduces maps in FIFO order, by N SUs. This is better than turning
178 /// every Nth memory SU into BarrierChain in buildSchedGraph(), since
179 /// it avoids unnecessary edges between seen SUs above the new BarrierChain,
180 /// and those below it.
181 void reduceHugeMemNodeMaps(Value2SUsMap &stores,
182 Value2SUsMap &loads, unsigned N);
184 /// \brief Adds a chain edge between SUa and SUb, but only if both
185 /// AliasAnalysis and Target fail to deny the dependency.
186 void addChainDependency(SUnit *SUa, SUnit *SUb,
187 unsigned Latency = 0);
189 /// Adds dependencies as needed from all SUs in list to SU.
190 void addChainDependencies(SUnit *SU, SUList &SUs, unsigned Latency) {
191 for (SUnit *Entry : SUs)
192 addChainDependency(SU, Entry, Latency);
195 /// Adds dependencies as needed from all SUs in map, to SU.
196 void addChainDependencies(SUnit *SU, Value2SUsMap &Val2SUsMap);
198 /// Adds dependencies as needed to SU, from all SUs mapped to V.
199 void addChainDependencies(SUnit *SU, Value2SUsMap &Val2SUsMap,
202 /// Adds barrier chain edges from all SUs in map, and then clear the map.
203 /// This is equivalent to insertBarrierChain(), but optimized for the common
204 /// case where the new BarrierChain (a global memory object) has a higher
205 /// NodeNum than all SUs in map. It is assumed BarrierChain has been set
206 /// before calling this.
207 void addBarrierChain(Value2SUsMap &map);
209 /// Inserts a barrier chain in a huge region, far below current SU.
210 /// Adds barrier chain edges from all SUs in map with higher NodeNums than
211 /// this new BarrierChain, and remove them from map. It is assumed
212 /// BarrierChain has been set before calling this.
213 void insertBarrierChain(Value2SUsMap &map);
215 /// For an unanalyzable memory access, this Value is used in maps.
216 UndefValue *UnknownValue;
218 typedef std::vector<std::pair<MachineInstr *, MachineInstr *>>
220 /// Remember instruction that precedes DBG_VALUE.
221 /// These are generated by buildSchedGraph but persist so they can be
222 /// referenced when emitting the final schedule.
223 DbgValueVector DbgValues;
224 MachineInstr *FirstDbgValue;
226 /// Set of live physical registers for updating kill flags.
230 explicit ScheduleDAGInstrs(MachineFunction &mf,
231 const MachineLoopInfo *mli,
232 bool RemoveKillFlags = false);
234 ~ScheduleDAGInstrs() override {}
236 /// Gets the machine model for instruction scheduling.
237 const TargetSchedModel *getSchedModel() const { return &SchedModel; }
239 /// Resolves and cache a resolved scheduling class for an SUnit.
240 const MCSchedClassDesc *getSchedClass(SUnit *SU) const {
241 if (!SU->SchedClass && SchedModel.hasInstrSchedModel())
242 SU->SchedClass = SchedModel.resolveSchedClass(SU->getInstr());
243 return SU->SchedClass;
246 /// Returns an iterator to the top of the current scheduling region.
247 MachineBasicBlock::iterator begin() const { return RegionBegin; }
249 /// Returns an iterator to the bottom of the current scheduling region.
250 MachineBasicBlock::iterator end() const { return RegionEnd; }
252 /// Creates a new SUnit and return a ptr to it.
253 SUnit *newSUnit(MachineInstr *MI);
255 /// Returns an existing SUnit for this MI, or nullptr.
256 SUnit *getSUnit(MachineInstr *MI) const;
258 /// Prepares to perform scheduling in the given block.
259 virtual void startBlock(MachineBasicBlock *BB);
261 /// Cleans up after scheduling in the given block.
262 virtual void finishBlock();
264 /// \brief Initialize the DAG and common scheduler state for a new
265 /// scheduling region. This does not actually create the DAG, only clears
266 /// it. The scheduling driver may call BuildSchedGraph multiple times per
267 /// scheduling region.
268 virtual void enterRegion(MachineBasicBlock *bb,
269 MachineBasicBlock::iterator begin,
270 MachineBasicBlock::iterator end,
271 unsigned regioninstrs);
273 /// Called when the scheduler has finished scheduling the current region.
274 virtual void exitRegion();
276 /// Builds SUnits for the current region.
277 /// If \p RPTracker is non-null, compute register pressure as a side effect.
278 /// The DAG builder is an efficient place to do it because it already visits
280 void buildSchedGraph(AliasAnalysis *AA,
281 RegPressureTracker *RPTracker = nullptr,
282 PressureDiffs *PDiffs = nullptr,
283 LiveIntervals *LIS = nullptr,
284 bool TrackLaneMasks = false);
286 /// \brief Adds dependencies from instructions in the current list of
287 /// instructions being scheduled to scheduling barrier. We want to make sure
288 /// instructions which define registers that are either used by the
289 /// terminator or are live-out are properly scheduled. This is especially
290 /// important when the definition latency of the return value(s) are too
291 /// high to be hidden by the branch or when the liveout registers used by
292 /// instructions in the fallthrough block.
293 void addSchedBarrierDeps();
295 /// Orders nodes according to selected style.
297 /// Typically, a scheduling algorithm will implement schedule() without
298 /// overriding enterRegion() or exitRegion().
299 virtual void schedule() = 0;
301 /// Allow targets to perform final scheduling actions at the level of the
302 /// whole MachineFunction. By default does nothing.
303 virtual void finalizeSchedule() {}
305 void dumpNode(const SUnit *SU) const override;
307 /// Returns a label for a DAG node that points to an instruction.
308 std::string getGraphNodeLabel(const SUnit *SU) const override;
310 /// Returns a label for the region of code covered by the DAG.
311 std::string getDAGName() const override;
313 /// Fixes register kill flags that scheduling has made invalid.
314 void fixupKills(MachineBasicBlock *MBB);
318 void addPhysRegDataDeps(SUnit *SU, unsigned OperIdx);
319 void addPhysRegDeps(SUnit *SU, unsigned OperIdx);
320 void addVRegDefDeps(SUnit *SU, unsigned OperIdx);
321 void addVRegUseDeps(SUnit *SU, unsigned OperIdx);
323 /// Initializes register live-range state for updating kills.
324 /// PostRA helper for rewriting kill flags.
325 void startBlockForKills(MachineBasicBlock *BB);
327 /// Toggles a register operand kill flag.
329 /// Other adjustments may be made to the instruction if necessary. Return
330 /// true if the operand has been deleted, false if not.
331 void toggleKillFlag(MachineInstr &MI, MachineOperand &MO);
333 /// Returns a mask for which lanes get read/written by the given (register)
335 LaneBitmask getLaneMaskForMO(const MachineOperand &MO) const;
338 /// Creates a new SUnit and return a ptr to it.
339 inline SUnit *ScheduleDAGInstrs::newSUnit(MachineInstr *MI) {
341 const SUnit *Addr = SUnits.empty() ? nullptr : &SUnits[0];
343 SUnits.emplace_back(MI, (unsigned)SUnits.size());
344 assert((Addr == nullptr || Addr == &SUnits[0]) &&
345 "SUnits std::vector reallocated on the fly!");
346 return &SUnits.back();
349 /// Returns an existing SUnit for this MI, or nullptr.
350 inline SUnit *ScheduleDAGInstrs::getSUnit(MachineInstr *MI) const {
351 DenseMap<MachineInstr*, SUnit*>::const_iterator I = MISUnitMap.find(MI);
352 if (I == MISUnitMap.end())
356 } // end namespace llvm