2 .\" Copyright (C) 2018 Matthew Macy <mmacy@FreeBSD.org>.
4 .\" Redistribution and use in source and binary forms, with or without
5 .\" modification, are permitted provided that the following conditions
7 .\" 1. Redistributions of source code must retain the above copyright
8 .\" notice(s), this list of conditions and the following disclaimer as
9 .\" the first lines of this file unmodified other than the possible
10 .\" addition of one or more copyright notices.
11 .\" 2. Redistributions in binary form must reproduce the above copyright
12 .\" notice(s), this list of conditions and the following disclaimer in the
13 .\" documentation and/or other materials provided with the distribution.
15 .\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
16 .\" EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 .\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 .\" DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
19 .\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 .\" (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
21 .\" SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
22 .\" CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
40 .Nm epoch_enter_preempt ,
41 .Nm epoch_exit_preempt ,
42 .Nm epoch_wait_preempt ,
44 .Nm epoch_drain_callbacks ,
47 .Nd kernel epoch based reclamation
54 struct epoch; /* Opaque */
56 .Vt typedef "struct epoch *epoch_t" ;
58 struct epoch_context {
62 .Vt typedef "struct epoch_context *epoch_context_t" ;
63 .Vt typedef "void epoch_callback_t(epoch_context_t)" ;
65 struct epoch_tracker; /* Opaque */
67 .Vt typedef "struct epoch_tracker *epoch_tracker_t" ;
70 .Fn epoch_alloc "const char *name" "int flags"
72 .Fn epoch_free "epoch_t epoch"
74 .Fn epoch_enter "epoch_t epoch"
76 .Fn epoch_exit "epoch_t epoch"
78 .Fn epoch_wait "epoch_t epoch"
80 .Fn epoch_enter_preempt "epoch_t epoch" "epoch_tracker_t et"
82 .Fn epoch_exit_preempt "epoch_t epoch" "epoch_tracker_t et"
84 .Fn epoch_wait_preempt "epoch_t epoch"
86 .Fn epoch_call "epoch_t epoch" "epoch_callback_t callback" "epoch_context_t ctx"
88 .Fn epoch_drain_callbacks "epoch_t epoch"
90 .Fn in_epoch "epoch_t epoch"
92 .Fn in_epoch_verbose "epoch_t epoch" "int dump_onfail"
94 Epochs are used to guarantee liveness and immutability of data by
95 deferring reclamation and mutation until a grace period has elapsed.
96 Epochs do not have any lock ordering issues.
97 Entering and leaving an epoch section will never block.
99 Epochs are allocated with
103 argument is used for debugging convenience when the
105 kernel option is configured.
106 By default, epochs do not allow preemption during sections.
107 By default mutexes cannot be held across
108 .Fn epoch_wait_preempt .
111 specified are formed by
113 the following values:
114 .Bl -tag -offset indent -width Ds
116 Permit holding mutexes across
117 .Fn epoch_wait_preempt
119 .Dv EPOCH_PREEMPT ) .
120 When doing this one must be cautious of creating a situation where a deadlock
125 will allow preemption during sections.
126 Only non-sleepable locks may be acquired during a preemptible epoch.
128 .Fn epoch_enter_preempt ,
129 .Fn epoch_exit_preempt ,
131 .Fn epoch_wait_preempt
132 must be used in place of
144 Threads indicate the start of an epoch critical section by calling
147 .Fn epoch_enter_preempt
148 for preemptible epochs).
152 .Fn epoch_exit_preempt
153 for preemptible epochs)
154 to indicate the end of a critical section.
155 .Vt struct epoch_tracker Ns s
156 are stack objects whose pointers are passed to
157 .Fn epoch_enter_preempt
159 .Fn epoch_exit_preempt
161 .Vt struct rm_priotracker ) .
163 Threads can defer work until a grace period has expired since any thread has
164 entered the epoch either synchronously or asynchronously.
166 defers work asynchronously by invoking the provided
171 .Fn epoch_wait_preempt )
172 blocks the current thread until the grace period has expired and the work can be
175 Default, non-preemptible epoch wait
177 is guaranteed to have much shorter completion times relative to
178 preemptible epoch wait
179 .Fn ( epoch_wait_preempt ) .
180 (In the default type, none of the threads in an epoch section will be preempted
181 before completing its section.)
183 INVARIANTS can assert that a thread is in an epoch by using
186 is equivalent to invoking
187 .Fn in_epoch_verbose "epoch" "0" .
191 .Fn in_epoch_verbose "epoch" "1"
192 provides additional verbose debugging information.
194 The epoch API currently does not support sleeping in epoch_preempt sections.
195 A caller should never call
197 in the middle of an epoch section for the same epoch as this will lead to a deadlock.
200 .Fn epoch_drain_callbacks
201 function is used to drain all pending callbacks which have been invoked by prior
203 function calls on the same epoch.
204 This function is useful when there are shared memory structure(s)
205 referred to by the epoch callback(s) which are not refcounted and are
207 The typical place for calling this function is right before freeing or
208 invalidating the shared resource(s) used by the epoch callback(s).
209 This function can sleep and is not optimized for performance.
211 .Fn in_epoch curepoch
212 will return 1 if curthread is in curepoch, 0 otherwise.
218 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_laddr *laddr,
222 epoch_enter(net_epoch);
223 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
225 if (sa->sa_family != AF_INET)
227 sin = (struct sockaddr_in *)sa;
228 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
229 ia = (struct in_ifaddr *)ifa;
233 epoch_exit(net_epoch);
240 ifa_free(struct ifaddr *ifa)
243 if (refcount_release(&ifa->ifa_refcnt))
244 epoch_call(net_epoch, ifa_destroy, &ifa->ifa_epoch_ctx);
248 if_purgeaddrs(struct ifnet *ifp)
253 CK_STAILQ_REMOVE(&ifp->if_addrhead, ifa, ifaddr, ifa_link);
254 IF_ADDR_WUNLOCK(ifp);
259 Thread 1 traverses the ifaddr list in an epoch.
260 Thread 2 unlinks with the corresponding epoch safe macro, marks as logically free,
261 and then defers deletion.
262 More general mutation or a synchronous
263 free would have to follow a call to
268 kernel programming interface is under development and is subject to change.
281 framework first appeared in
284 One must be cautious when using
285 .Fn epoch_wait_preempt .
286 Threads are pinned during epoch sections, so if a thread in a section is then
287 preempted by a higher priority compute bound thread on that CPU, it can be
288 prevented from leaving the section indefinitely.
290 Epochs are not a straight replacement for read locks.
291 Callers must use safe list and tailq traversal routines in an epoch (see ck_queue).
292 When modifying a list referenced from an epoch section safe removal
293 routines must be used and the caller can no longer modify a list entry
295 An item to be modified must be handled with copy on write
296 and frees must be deferred until after a grace period has elapsed.