2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2017, Jeffrey Roberson <jeff@freebsd.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/bitset.h>
38 #include <sys/domainset.h>
41 #include <sys/mutex.h>
42 #include <sys/malloc.h>
43 #include <sys/vmmeter.h>
46 #include <vm/vm_param.h>
47 #include <vm/vm_domainset.h>
48 #include <vm/vm_object.h>
49 #include <vm/vm_page.h>
50 #include <vm/vm_phys.h>
54 * Iterators are written such that the first nowait pass has as short a
55 * codepath as possible to eliminate bloat from the allocator. It is
56 * assumed that most allocations are successful.
60 * Determine which policy is to be used for this allocation.
63 vm_domainset_iter_domain(struct vm_domainset_iter *di, struct vm_object *obj)
65 struct domainset *domain;
68 * object policy takes precedence over thread policy. The policies
69 * are immutable and unsynchronized. Updates can race but pointer
70 * loads are assumed to be atomic.
72 if (obj != NULL && (domain = obj->domain.dr_policy) != NULL) {
73 di->di_domain = domain;
74 di->di_iter = &obj->domain.dr_iterator;
76 di->di_domain = curthread->td_domain.dr_policy;
77 di->di_iter = &curthread->td_domain.dr_iterator;
82 vm_domainset_iter_rr(struct vm_domainset_iter *di, int *domain)
88 d = (d + 1) % di->di_domain->ds_max;
89 } while (!DOMAINSET_ISSET(d, &di->di_domain->ds_mask));
90 *di->di_iter = *domain = d;
94 vm_domainset_iter_prefer(struct vm_domainset_iter *di, int *domain)
100 d = (d + 1) % di->di_domain->ds_max;
101 } while (!DOMAINSET_ISSET(d, &di->di_domain->ds_mask) ||
102 d == di->di_domain->ds_prefer);
103 *di->di_iter = *domain = d;
107 vm_domainset_iter_next(struct vm_domainset_iter *di, int *domain)
110 KASSERT(di->di_n > 0,
111 ("vm_domainset_iter_first: Invalid n %d", di->di_n));
112 switch (di->di_domain->ds_policy) {
113 case DOMAINSET_POLICY_FIRSTTOUCH:
115 * To prevent impossible allocations we convert an invalid
116 * first-touch to round-robin.
119 case DOMAINSET_POLICY_ROUNDROBIN:
120 vm_domainset_iter_rr(di, domain);
122 case DOMAINSET_POLICY_PREFER:
123 vm_domainset_iter_prefer(di, domain);
126 panic("vm_domainset_iter_first: Unknown policy %d",
127 di->di_domain->ds_policy);
129 KASSERT(*domain < vm_ndomains,
130 ("vm_domainset_iter_next: Invalid domain %d", *domain));
134 vm_domainset_iter_first(struct vm_domainset_iter *di, int *domain)
137 switch (di->di_domain->ds_policy) {
138 case DOMAINSET_POLICY_FIRSTTOUCH:
139 *domain = PCPU_GET(domain);
140 if (DOMAINSET_ISSET(*domain, &di->di_domain->ds_mask)) {
145 * To prevent impossible allocations we convert an invalid
146 * first-touch to round-robin.
149 case DOMAINSET_POLICY_ROUNDROBIN:
150 di->di_n = di->di_domain->ds_cnt;
151 vm_domainset_iter_rr(di, domain);
153 case DOMAINSET_POLICY_PREFER:
154 *domain = di->di_domain->ds_prefer;
155 di->di_n = di->di_domain->ds_cnt;
158 panic("vm_domainset_iter_first: Unknown policy %d",
159 di->di_domain->ds_policy);
161 KASSERT(di->di_n > 0,
162 ("vm_domainset_iter_first: Invalid n %d", di->di_n));
163 KASSERT(*domain < vm_ndomains,
164 ("vm_domainset_iter_first: Invalid domain %d", *domain));
168 vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
169 int *domain, int *req)
172 vm_domainset_iter_domain(di, obj);
174 *req = (di->di_flags & ~(VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) |
176 vm_domainset_iter_first(di, domain);
180 vm_domainset_iter_page(struct vm_domainset_iter *di, int *domain, int *req)
184 * If we exhausted all options with NOWAIT and did a WAITFAIL it
185 * is time to return an error to the caller.
187 if ((*req & VM_ALLOC_WAITFAIL) != 0)
190 /* If there are more domains to visit we run the iterator. */
191 if (--di->di_n != 0) {
192 vm_domainset_iter_next(di, domain);
196 /* If we visited all domains and this was a NOWAIT we return error. */
197 if ((di->di_flags & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) == 0)
201 * We have visited all domains with non-blocking allocations, try
202 * from the beginning with a blocking allocation.
204 vm_domainset_iter_first(di, domain);
212 vm_domainset_iter_malloc_init(struct vm_domainset_iter *di,
213 struct vm_object *obj, int *domain, int *flags)
216 vm_domainset_iter_domain(di, obj);
217 di->di_flags = *flags;
218 *flags = (di->di_flags & ~M_WAITOK) | M_NOWAIT;
219 vm_domainset_iter_first(di, domain);
223 vm_domainset_iter_malloc(struct vm_domainset_iter *di, int *domain, int *flags)
226 /* If there are more domains to visit we run the iterator. */
227 if (--di->di_n != 0) {
228 vm_domainset_iter_next(di, domain);
232 /* If we visited all domains and this was a NOWAIT we return error. */
233 if ((di->di_flags & M_WAITOK) == 0)
237 * We have visited all domains with non-blocking allocations, try
238 * from the beginning with a blocking allocation.
240 vm_domainset_iter_first(di, domain);
241 *flags = di->di_flags;
248 vm_domainset_iter_page(struct vm_domainset_iter *di, int *domain, int *flags)
251 return (EJUSTRETURN);
255 vm_domainset_iter_page_init(struct vm_domainset_iter *di,
256 struct vm_object *obj, int *domain, int *flags)
263 vm_domainset_iter_malloc(struct vm_domainset_iter *di, int *domain, int *flags)
266 return (EJUSTRETURN);
270 vm_domainset_iter_malloc_init(struct vm_domainset_iter *di,
271 struct vm_object *obj, int *domain, int *flags)