1 //===-- JITLinkMemoryManager.h - JITLink mem manager interface --*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // Contains the JITLinkMemoryManager interface.
11 //===----------------------------------------------------------------------===//
13 #ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINKMEMORYMANAGER_H
14 #define LLVM_EXECUTIONENGINE_JITLINK_JITLINKMEMORYMANAGER_H
16 #include "llvm/ADT/FunctionExtras.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ExecutionEngine/JITLink/JITLinkDylib.h"
19 #include "llvm/ExecutionEngine/JITLink/MemoryFlags.h"
20 #include "llvm/ExecutionEngine/Orc/Shared/AllocationActions.h"
21 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
22 #include "llvm/Support/Allocator.h"
23 #include "llvm/Support/Error.h"
24 #include "llvm/Support/MSVCErrorWorkarounds.h"
25 #include "llvm/Support/Memory.h"
26 #include "llvm/Support/RecyclingAllocator.h"
39 /// Manages allocations of JIT memory.
41 /// Instances of this class may be accessed concurrently from multiple threads
42 /// and their implemetations should include any necessary synchronization.
43 class JITLinkMemoryManager {
46 /// Represents a finalized allocation.
48 /// Finalized allocations must be passed to the
49 /// JITLinkMemoryManager:deallocate method prior to being destroyed.
51 /// The interpretation of the Address associated with the finalized allocation
52 /// is up to the memory manager implementation. Common options are using the
53 /// base address of the allocation, or the address of a memory management
54 /// object that tracks the allocation.
55 class FinalizedAlloc {
56 friend class JITLinkMemoryManager;
58 static constexpr auto InvalidAddr = ~uint64_t(0);
61 FinalizedAlloc() = default;
62 explicit FinalizedAlloc(orc::ExecutorAddr A) : A(A) {
63 assert(A.getValue() != InvalidAddr &&
64 "Explicitly creating an invalid allocation?");
66 FinalizedAlloc(const FinalizedAlloc &) = delete;
67 FinalizedAlloc(FinalizedAlloc &&Other) : A(Other.A) {
68 Other.A.setValue(InvalidAddr);
70 FinalizedAlloc &operator=(const FinalizedAlloc &) = delete;
71 FinalizedAlloc &operator=(FinalizedAlloc &&Other) {
72 assert(A.getValue() == InvalidAddr &&
73 "Cannot overwrite active finalized allocation");
74 std::swap(A, Other.A);
78 assert(A.getValue() == InvalidAddr &&
79 "Finalized allocation was not deallocated");
82 /// FinalizedAllocs convert to false for default-constructed, and
83 /// true otherwise. Default-constructed allocs need not be deallocated.
84 explicit operator bool() const { return A.getValue() != InvalidAddr; }
86 /// Returns the address associated with this finalized allocation.
87 /// The allocation is unmodified.
88 orc::ExecutorAddr getAddress() const { return A; }
90 /// Returns the address associated with this finalized allocation and
91 /// resets this object to the default state.
92 /// This should only be used by allocators when deallocating memory.
93 orc::ExecutorAddr release() {
94 orc::ExecutorAddr Tmp = A;
95 A.setValue(InvalidAddr);
100 orc::ExecutorAddr A{InvalidAddr};
103 /// Represents an allocation which has not been finalized yet.
105 /// InFlightAllocs manage both executor memory allocations and working
106 /// memory allocations.
108 /// On finalization, the InFlightAlloc should transfer the content of
109 /// working memory into executor memory, apply memory protections, and
110 /// run any finalization functions.
112 /// Working memory should be kept alive at least until one of the following
113 /// happens: (1) the InFlightAlloc instance is destroyed, (2) the
114 /// InFlightAlloc is abandoned, (3) finalized target memory is destroyed.
116 /// If abandon is called then working memory and executor memory should both
118 class InFlightAlloc {
120 using OnFinalizedFunction = unique_function<void(Expected<FinalizedAlloc>)>;
121 using OnAbandonedFunction = unique_function<void(Error)>;
123 virtual ~InFlightAlloc();
125 /// Called prior to finalization if the allocation should be abandoned.
126 virtual void abandon(OnAbandonedFunction OnAbandoned) = 0;
128 /// Called to transfer working memory to the target and apply finalization.
129 virtual void finalize(OnFinalizedFunction OnFinalized) = 0;
131 /// Synchronous convenience version of finalize.
132 Expected<FinalizedAlloc> finalize() {
133 std::promise<MSVCPExpected<FinalizedAlloc>> FinalizeResultP;
134 auto FinalizeResultF = FinalizeResultP.get_future();
135 finalize([&](Expected<FinalizedAlloc> Result) {
136 FinalizeResultP.set_value(std::move(Result));
138 return FinalizeResultF.get();
142 /// Typedef for the argument to be passed to OnAllocatedFunction.
143 using AllocResult = Expected<std::unique_ptr<InFlightAlloc>>;
145 /// Called when allocation has been completed.
146 using OnAllocatedFunction = unique_function<void(AllocResult)>;
148 /// Called when deallocation has completed.
149 using OnDeallocatedFunction = unique_function<void(Error)>;
151 virtual ~JITLinkMemoryManager();
153 /// Start the allocation process.
155 /// If the initial allocation is successful then the OnAllocated function will
156 /// be called with a std::unique_ptr<InFlightAlloc> value. If the assocation
157 /// is unsuccessful then the OnAllocated function will be called with an
159 virtual void allocate(const JITLinkDylib *JD, LinkGraph &G,
160 OnAllocatedFunction OnAllocated) = 0;
162 /// Convenience function for blocking allocation.
163 AllocResult allocate(const JITLinkDylib *JD, LinkGraph &G) {
164 std::promise<MSVCPExpected<std::unique_ptr<InFlightAlloc>>> AllocResultP;
165 auto AllocResultF = AllocResultP.get_future();
166 allocate(JD, G, [&](AllocResult Alloc) {
167 AllocResultP.set_value(std::move(Alloc));
169 return AllocResultF.get();
172 /// Deallocate a list of allocation objects.
174 /// Dealloc actions will be run in reverse order (from the end of the vector
176 virtual void deallocate(std::vector<FinalizedAlloc> Allocs,
177 OnDeallocatedFunction OnDeallocated) = 0;
179 /// Convenience function for deallocation of a single alloc.
180 void deallocate(FinalizedAlloc Alloc, OnDeallocatedFunction OnDeallocated) {
181 std::vector<FinalizedAlloc> Allocs;
182 Allocs.push_back(std::move(Alloc));
183 deallocate(std::move(Allocs), std::move(OnDeallocated));
186 /// Convenience function for blocking deallocation.
187 Error deallocate(std::vector<FinalizedAlloc> Allocs) {
188 std::promise<MSVCPError> DeallocResultP;
189 auto DeallocResultF = DeallocResultP.get_future();
190 deallocate(std::move(Allocs),
191 [&](Error Err) { DeallocResultP.set_value(std::move(Err)); });
192 return DeallocResultF.get();
195 /// Convenience function for blocking deallocation of a single alloc.
196 Error deallocate(FinalizedAlloc Alloc) {
197 std::vector<FinalizedAlloc> Allocs;
198 Allocs.push_back(std::move(Alloc));
199 return deallocate(std::move(Allocs));
203 /// BasicLayout simplifies the implementation of JITLinkMemoryManagers.
205 /// BasicLayout groups Sections into Segments based on their memory protection
206 /// and deallocation policies. JITLinkMemoryManagers can construct a BasicLayout
207 /// from a Graph, and then assign working memory and addresses to each of the
208 /// Segments. These addreses will be mapped back onto the Graph blocks in
209 /// the apply method.
212 /// The Alignment, ContentSize and ZeroFillSize of each segment will be
213 /// pre-filled from the Graph. Clients must set the Addr and WorkingMem fields
214 /// prior to calling apply.
216 // FIXME: The C++98 initializer is an attempt to work around compile failures
217 // due to http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#1397.
218 // We should be able to switch this back to member initialization once that
221 friend class BasicLayout;
225 : ContentSize(0), ZeroFillSize(0), Addr(0), WorkingMem(nullptr),
226 NextWorkingMemOffset(0) {}
229 uint64_t ZeroFillSize;
230 orc::ExecutorAddr Addr;
231 char *WorkingMem = nullptr;
234 size_t NextWorkingMemOffset;
235 std::vector<Block *> ContentBlocks, ZeroFillBlocks;
238 /// A convenience class that further groups segments based on memory
239 /// deallocation policy. This allows clients to make two slab allocations:
240 /// one for all standard segments, and one for all finalize segments.
241 struct ContiguousPageBasedLayoutSizes {
242 uint64_t StandardSegs = 0;
243 uint64_t FinalizeSegs = 0;
245 uint64_t total() const { return StandardSegs + FinalizeSegs; }
249 using SegmentMap = AllocGroupSmallMap<Segment>;
252 BasicLayout(LinkGraph &G);
254 /// Return a reference to the graph this allocation was created from.
255 LinkGraph &getGraph() { return G; }
257 /// Returns the total number of required to allocate all segments (with each
258 /// segment padded out to page size) for all standard segments, and all
259 /// finalize segments.
261 /// This is a convenience function for the common case where the segments will
262 /// be allocated contiguously.
264 /// This function will return an error if any segment has an alignment that
265 /// is higher than a page.
266 Expected<ContiguousPageBasedLayoutSizes>
267 getContiguousPageBasedLayoutSizes(uint64_t PageSize);
269 /// Returns an iterator over the segments of the layout.
270 iterator_range<SegmentMap::iterator> segments() {
271 return {Segments.begin(), Segments.end()};
274 /// Apply the layout to the graph.
277 /// Returns a reference to the AllocActions in the graph.
278 /// This convenience function saves callers from having to #include
279 /// LinkGraph.h if all they need are allocation actions.
280 orc::shared::AllocActions &graphAllocActions();
287 /// A utility class for making simple allocations using JITLinkMemoryManager.
289 /// SimpleSegementAlloc takes a mapping of AllocGroups to Segments and uses
290 /// this to create a LinkGraph with one Section (containing one Block) per
291 /// Segment. Clients can obtain a pointer to the working memory and executor
292 /// address of that block using the Segment's AllocGroup. Once memory has been
293 /// populated, clients can call finalize to finalize the memory.
294 class SimpleSegmentAlloc {
296 /// Describes a segment to be allocated.
299 Segment(size_t ContentSize, Align ContentAlign)
300 : ContentSize(ContentSize), ContentAlign(ContentAlign) {}
302 size_t ContentSize = 0;
306 /// Describes the segment working memory and executor address.
308 orc::ExecutorAddr Addr;
309 MutableArrayRef<char> WorkingMem;
312 using SegmentMap = AllocGroupSmallMap<Segment>;
314 using OnCreatedFunction = unique_function<void(Expected<SimpleSegmentAlloc>)>;
316 using OnFinalizedFunction =
317 JITLinkMemoryManager::InFlightAlloc::OnFinalizedFunction;
319 static void Create(JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD,
320 SegmentMap Segments, OnCreatedFunction OnCreated);
322 static Expected<SimpleSegmentAlloc> Create(JITLinkMemoryManager &MemMgr,
323 const JITLinkDylib *JD,
324 SegmentMap Segments);
326 SimpleSegmentAlloc(SimpleSegmentAlloc &&);
327 SimpleSegmentAlloc &operator=(SimpleSegmentAlloc &&);
328 ~SimpleSegmentAlloc();
330 /// Returns the SegmentInfo for the given group.
331 SegmentInfo getSegInfo(AllocGroup AG);
333 /// Finalize all groups (async version).
334 void finalize(OnFinalizedFunction OnFinalized) {
335 Alloc->finalize(std::move(OnFinalized));
338 /// Finalize all groups.
339 Expected<JITLinkMemoryManager::FinalizedAlloc> finalize() {
340 return Alloc->finalize();
345 std::unique_ptr<LinkGraph> G, AllocGroupSmallMap<Block *> ContentBlocks,
346 std::unique_ptr<JITLinkMemoryManager::InFlightAlloc> Alloc);
348 std::unique_ptr<LinkGraph> G;
349 AllocGroupSmallMap<Block *> ContentBlocks;
350 std::unique_ptr<JITLinkMemoryManager::InFlightAlloc> Alloc;
353 /// A JITLinkMemoryManager that allocates in-process memory.
354 class InProcessMemoryManager : public JITLinkMemoryManager {
356 class IPInFlightAlloc;
358 /// Attempts to auto-detect the host page size.
359 static Expected<std::unique_ptr<InProcessMemoryManager>> Create();
361 /// Create an instance using the given page size.
362 InProcessMemoryManager(uint64_t PageSize) : PageSize(PageSize) {}
364 void allocate(const JITLinkDylib *JD, LinkGraph &G,
365 OnAllocatedFunction OnAllocated) override;
367 // Use overloads from base class.
368 using JITLinkMemoryManager::allocate;
370 void deallocate(std::vector<FinalizedAlloc> Alloc,
371 OnDeallocatedFunction OnDeallocated) override;
373 // Use overloads from base class.
374 using JITLinkMemoryManager::deallocate;
377 // FIXME: Use an in-place array instead of a vector for DeallocActions.
378 // There shouldn't need to be a heap alloc for this.
379 struct FinalizedAllocInfo {
380 sys::MemoryBlock StandardSegments;
381 std::vector<orc::shared::WrapperFunctionCall> DeallocActions;
384 FinalizedAlloc createFinalizedAlloc(
385 sys::MemoryBlock StandardSegments,
386 std::vector<orc::shared::WrapperFunctionCall> DeallocActions);
389 std::mutex FinalizedAllocsMutex;
390 RecyclingAllocator<BumpPtrAllocator, FinalizedAllocInfo> FinalizedAllocInfos;
393 } // end namespace jitlink
394 } // end namespace llvm
396 #endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINKMEMORYMANAGER_H