sys/vm/vm_domain.c

   1 /*-
   2  * Copyright (c) 2015 Adrian Chadd <adrian@FreeBSD.org>.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer,
  10  *    without modification.
  11  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  12  *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
  13  *    redistribution must be conditioned upon including a substantially
  14  *    similar Disclaimer requirement for further binary redistribution.
  15  *
  16  * NO WARRANTY
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19  * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
  20  * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
  22  * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  23  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  25  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  26  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  27  * THE POSSIBILITY OF SUCH DAMAGES.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include "opt_vm.h"
  34 #include "opt_ddb.h"
  35
  36 #include <sys/param.h>
  37 #include <sys/systm.h>
  38 #include <sys/lock.h>
  39 #include <sys/kernel.h>
  40 #include <sys/malloc.h>
  41 #include <sys/mutex.h>
  42 #ifdef VM_NUMA_ALLOC
  43 #include <sys/proc.h>
  44 #endif
  45 #include <sys/queue.h>
  46 #include <sys/rwlock.h>
  47 #include <sys/sbuf.h>
  48 #include <sys/sysctl.h>
  49 #include <sys/tree.h>
  50 #include <sys/vmmeter.h>
  51 #include <sys/seq.h>
  52
  53 #include <ddb/ddb.h>
  54
  55 #include <vm/vm.h>
  56 #include <vm/vm_param.h>
  57 #include <vm/vm_kern.h>
  58 #include <vm/vm_object.h>
  59 #include <vm/vm_page.h>
  60 #include <vm/vm_phys.h>
  61
  62 #include <vm/vm_domain.h>
  63
  64 #ifdef VM_NUMA_ALLOC
  65 static __inline int
  66 vm_domain_rr_selectdomain(int skip_domain)
  67 {
  68         struct thread *td;
  69
  70         td = curthread;
  71
  72         td->td_dom_rr_idx++;
  73         td->td_dom_rr_idx %= vm_ndomains;
  74
  75         /*
  76          * If skip_domain is provided then skip over that
  77          * domain.  This is intended for round robin variants
  78          * which first try a fixed domain.
  79          */
  80         if ((skip_domain > -1) && (td->td_dom_rr_idx == skip_domain)) {
  81                 td->td_dom_rr_idx++;
  82                 td->td_dom_rr_idx %= vm_ndomains;
  83         }
  84         return (td->td_dom_rr_idx);
  85 }
  86 #endif
  87
  88 /*
  89  * This implements a very simple set of VM domain memory allocation
  90  * policies and iterators.
  91  */
  92
  93 /*
  94  * A VM domain policy represents a desired VM domain policy.
  95  * Iterators implement searching through VM domains in a specific
  96  * order.
  97  */
  98
  99 /*
 100  * When setting a policy, the caller must establish their own
 101  * exclusive write protection for the contents of the domain
 102  * policy.
 103  */
 104 int
 105 vm_domain_policy_init(struct vm_domain_policy *vp)
 106 {
 107
 108         bzero(vp, sizeof(*vp));
 109         vp->p.policy = VM_POLICY_NONE;
 110         vp->p.domain = -1;
 111         return (0);
 112 }
 113
 114 int
 115 vm_domain_policy_set(struct vm_domain_policy *vp,
 116     vm_domain_policy_type_t vt, int domain)
 117 {
 118
 119         seq_write_begin(&vp->seq);
 120         vp->p.policy = vt;
 121         vp->p.domain = domain;
 122         seq_write_end(&vp->seq);
 123         return (0);
 124 }
 125
 126 /*
 127  * Take a local copy of a policy.
 128  *
 129  * The destination policy isn't write-barriered; this is used
 130  * for doing local copies into something that isn't shared.
 131  */
 132 void
 133 vm_domain_policy_localcopy(struct vm_domain_policy *dst,
 134     const struct vm_domain_policy *src)
 135 {
 136         seq_t seq;
 137
 138         for (;;) {
 139                 seq = seq_read(&src->seq);
 140                 *dst = *src;
 141                 if (seq_consistent(&src->seq, seq))
 142                         return;
 143         }
 144 }
 145
 146 /*
 147  * Take a write-barrier copy of a policy.
 148  *
 149  * The destination policy is write -barriered; this is used
 150  * for doing copies into policies that may be read by other
 151  * threads.
 152  */
 153 void
 154 vm_domain_policy_copy(struct vm_domain_policy *dst,
 155     const struct vm_domain_policy *src)
 156 {
 157         seq_t seq;
 158         struct vm_domain_policy d;
 159
 160         for (;;) {
 161                 seq = seq_read(&src->seq);
 162                 d = *src;
 163                 if (seq_consistent(&src->seq, seq)) {
 164                         seq_write_begin(&dst->seq);
 165                         dst->p.domain = d.p.domain;
 166                         dst->p.policy = d.p.policy;
 167                         seq_write_end(&dst->seq);
 168                         return;
 169                 }
 170         }
 171 }
 172
 173 int
 174 vm_domain_policy_validate(const struct vm_domain_policy *vp)
 175 {
 176
 177         switch (vp->p.policy) {
 178         case VM_POLICY_NONE:
 179         case VM_POLICY_ROUND_ROBIN:
 180         case VM_POLICY_FIRST_TOUCH:
 181         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 182                 if (vp->p.domain == -1)
 183                         return (0);
 184                 return (-1);
 185         case VM_POLICY_FIXED_DOMAIN:
 186         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 187 #ifdef VM_NUMA_ALLOC
 188                 if (vp->p.domain >= 0 && vp->p.domain < vm_ndomains)
 189                         return (0);
 190 #else
 191                 if (vp->p.domain == 0)
 192                         return (0);
 193 #endif
 194                 return (-1);
 195         default:
 196                 return (-1);
 197         }
 198         return (-1);
 199 }
 200
 201 int
 202 vm_domain_policy_cleanup(struct vm_domain_policy *vp)
 203 {
 204
 205         /* For now, empty */
 206         return (0);
 207 }
 208
 209 int
 210 vm_domain_iterator_init(struct vm_domain_iterator *vi)
 211 {
 212
 213         /* Nothing to do for now */
 214         return (0);
 215 }
 216
 217 /*
 218  * Manually setup an iterator with the given details.
 219  */
 220 int
 221 vm_domain_iterator_set(struct vm_domain_iterator *vi,
 222     vm_domain_policy_type_t vt, int domain)
 223 {
 224
 225 #ifdef VM_NUMA_ALLOC
 226         switch (vt) {
 227         case VM_POLICY_FIXED_DOMAIN:
 228                 vi->policy = VM_POLICY_FIXED_DOMAIN;
 229                 vi->domain = domain;
 230                 vi->n = 1;
 231                 break;
 232         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 233                 vi->policy = VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN;
 234                 vi->domain = domain;
 235                 vi->n = vm_ndomains;
 236                 break;
 237         case VM_POLICY_FIRST_TOUCH:
 238                 vi->policy = VM_POLICY_FIRST_TOUCH;
 239                 vi->domain = PCPU_GET(domain);
 240                 vi->n = 1;
 241                 break;
 242         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 243                 vi->policy = VM_POLICY_FIRST_TOUCH_ROUND_ROBIN;
 244                 vi->domain = PCPU_GET(domain);
 245                 vi->n = vm_ndomains;
 246                 break;
 247         case VM_POLICY_ROUND_ROBIN:
 248         default:
 249                 vi->policy = VM_POLICY_ROUND_ROBIN;
 250                 vi->domain = -1;
 251                 vi->n = vm_ndomains;
 252                 break;
 253         }
 254 #else
 255         vi->domain = 0;
 256         vi->n = 1;
 257 #endif
 258         return (0);
 259 }
 260
 261 /*
 262  * Setup an iterator based on the given policy.
 263  */
 264 static inline void
 265 _vm_domain_iterator_set_policy(struct vm_domain_iterator *vi,
 266     const struct vm_domain_policy *vt)
 267 {
 268
 269 #ifdef VM_NUMA_ALLOC
 270         /*
 271          * Initialise the iterator.
 272          *
 273          * For first-touch, the initial domain is set
 274          * via the current thread CPU domain.
 275          *
 276          * For fixed-domain, it's assumed that the
 277          * caller has initialised the specific domain
 278          * it is after.
 279          */
 280         switch (vt->p.policy) {
 281         case VM_POLICY_FIXED_DOMAIN:
 282                 vi->policy = vt->p.policy;
 283                 vi->domain = vt->p.domain;
 284                 vi->n = 1;
 285                 break;
 286         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 287                 vi->policy = vt->p.policy;
 288                 vi->domain = vt->p.domain;
 289                 vi->n = vm_ndomains;
 290                 break;
 291         case VM_POLICY_FIRST_TOUCH:
 292                 vi->policy = vt->p.policy;
 293                 vi->domain = PCPU_GET(domain);
 294                 vi->n = 1;
 295                 break;
 296         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 297                 vi->policy = vt->p.policy;
 298                 vi->domain = PCPU_GET(domain);
 299                 vi->n = vm_ndomains;
 300                 break;
 301         case VM_POLICY_ROUND_ROBIN:
 302         default:
 303                 /*
 304                  * Default to round-robin policy.
 305                  */
 306                 vi->policy = VM_POLICY_ROUND_ROBIN;
 307                 vi->domain = -1;
 308                 vi->n = vm_ndomains;
 309                 break;
 310         }
 311 #else
 312         vi->domain = 0;
 313         vi->n = 1;
 314 #endif
 315 }
 316
 317 void
 318 vm_domain_iterator_set_policy(struct vm_domain_iterator *vi,
 319     const struct vm_domain_policy *vt)
 320 {
 321         seq_t seq;
 322         struct vm_domain_policy vt_lcl;
 323
 324         for (;;) {
 325                 seq = seq_read(&vt->seq);
 326                 vt_lcl = *vt;
 327                 if (seq_consistent(&vt->seq, seq)) {
 328                         _vm_domain_iterator_set_policy(vi, &vt_lcl);
 329                         return;
 330                 }
 331         }
 332 }
 333
 334 /*
 335  * Return the next VM domain to use.
 336  *
 337  * Returns 0 w/ domain set to the next domain to use, or
 338  * -1 to indicate no more domains are available.
 339  */
 340 int
 341 vm_domain_iterator_run(struct vm_domain_iterator *vi, int *domain)
 342 {
 343
 344         /* General catch-all */
 345         if (vi->n <= 0)
 346                 return (-1);
 347
 348 #ifdef VM_NUMA_ALLOC
 349         switch (vi->policy) {
 350         case VM_POLICY_FIXED_DOMAIN:
 351         case VM_POLICY_FIRST_TOUCH:
 352                 *domain = vi->domain;
 353                 vi->n--;
 354                 break;
 355         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 356         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 357                 /*
 358                  * XXX TODO: skip over the rr'ed domain
 359                  * if it equals the one we started with.
 360                  */
 361                 if (vi->n == vm_ndomains)
 362                         *domain = vi->domain;
 363                 else
 364                         *domain = vm_domain_rr_selectdomain(vi->domain);
 365                 vi->n--;
 366                 break;
 367         case VM_POLICY_ROUND_ROBIN:
 368         default:
 369                 *domain = vm_domain_rr_selectdomain(-1);
 370                 vi->n--;
 371                 break;
 372         }
 373 #else
 374         *domain = 0;
 375         vi->n--;
 376 #endif
 377
 378         return (0);
 379 }
 380
 381 /*
 382  * Returns 1 if the iteration is done, or 0 if it has not.
 383
 384  * This can only be called after at least one loop through
 385  * the iterator.  Ie, it's designed to be used as a tail
 386  * check of a loop, not the head check of a loop.
 387  */
 388 int
 389 vm_domain_iterator_isdone(struct vm_domain_iterator *vi)
 390 {
 391
 392         return (vi->n <= 0);
 393 }
 394
 395 int
 396 vm_domain_iterator_cleanup(struct vm_domain_iterator *vi)
 397 {
 398
 399         return (0);
 400 }