sys/vm/vm_domain.c

   1 /*-
   2  * Copyright (c) 2015 Adrian Chadd <adrian@FreeBSD.org>.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer,
  10  *    without modification.
  11  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  12  *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
  13  *    redistribution must be conditioned upon including a substantially
  14  *    similar Disclaimer requirement for further binary redistribution.
  15  *
  16  * NO WARRANTY
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19  * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
  20  * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
  22  * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  23  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  25  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  26  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  27  * THE POSSIBILITY OF SUCH DAMAGES.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include "opt_vm.h"
  34 #include "opt_ddb.h"
  35
  36 #include <sys/param.h>
  37 #include <sys/systm.h>
  38 #include <sys/lock.h>
  39 #include <sys/kernel.h>
  40 #include <sys/malloc.h>
  41 #include <sys/mutex.h>
  42 #ifdef VM_NUMA_ALLOC
  43 #include <sys/proc.h>
  44 #endif
  45 #include <sys/queue.h>
  46 #include <sys/rwlock.h>
  47 #include <sys/sbuf.h>
  48 #include <sys/sysctl.h>
  49 #include <sys/tree.h>
  50 #include <sys/vmmeter.h>
  51 #include <sys/seq.h>
  52
  53 #include <ddb/ddb.h>
  54
  55 #include <vm/vm.h>
  56 #include <vm/vm_param.h>
  57 #include <vm/vm_kern.h>
  58 #include <vm/vm_object.h>
  59 #include <vm/vm_page.h>
  60 #include <vm/vm_phys.h>
  61
  62 #include <vm/vm_domain.h>
  63
  64 static __inline int
  65 vm_domain_rr_selectdomain(int skip_domain)
  66 {
  67 #ifdef VM_NUMA_ALLOC
  68         struct thread *td;
  69
  70         td = curthread;
  71
  72         td->td_dom_rr_idx++;
  73         td->td_dom_rr_idx %= vm_ndomains;
  74
  75         /*
  76          * If skip_domain is provided then skip over that
  77          * domain.  This is intended for round robin variants
  78          * which first try a fixed domain.
  79          */
  80         if ((skip_domain > -1) && (td->td_dom_rr_idx == skip_domain)) {
  81                 td->td_dom_rr_idx++;
  82                 td->td_dom_rr_idx %= vm_ndomains;
  83         }
  84         return (td->td_dom_rr_idx);
  85 #else
  86         return (0);
  87 #endif
  88 }
  89
  90 /*
  91  * This implements a very simple set of VM domain memory allocation
  92  * policies and iterators.
  93  */
  94
  95 /*
  96  * A VM domain policy represents a desired VM domain policy.
  97  * Iterators implement searching through VM domains in a specific
  98  * order.
  99  */
 100
 101 /*
 102  * When setting a policy, the caller must establish their own
 103  * exclusive write protection for the contents of the domain
 104  * policy.
 105  */
 106 int
 107 vm_domain_policy_init(struct vm_domain_policy *vp)
 108 {
 109
 110         bzero(vp, sizeof(*vp));
 111         vp->p.policy = VM_POLICY_NONE;
 112         vp->p.domain = -1;
 113         return (0);
 114 }
 115
 116 int
 117 vm_domain_policy_set(struct vm_domain_policy *vp,
 118     vm_domain_policy_type_t vt, int domain)
 119 {
 120
 121         seq_write_begin(&vp->seq);
 122         vp->p.policy = vt;
 123         vp->p.domain = domain;
 124         seq_write_end(&vp->seq);
 125         return (0);
 126 }
 127
 128 /*
 129  * Take a local copy of a policy.
 130  *
 131  * The destination policy isn't write-barriered; this is used
 132  * for doing local copies into something that isn't shared.
 133  */
 134 void
 135 vm_domain_policy_localcopy(struct vm_domain_policy *dst,
 136     const struct vm_domain_policy *src)
 137 {
 138         seq_t seq;
 139
 140         for (;;) {
 141                 seq = seq_read(&src->seq);
 142                 *dst = *src;
 143                 if (seq_consistent(&src->seq, seq))
 144                         return;
 145                 cpu_spinwait();
 146         }
 147 }
 148
 149 /*
 150  * Take a write-barrier copy of a policy.
 151  *
 152  * The destination policy is write -barriered; this is used
 153  * for doing copies into policies that may be read by other
 154  * threads.
 155  */
 156 void
 157 vm_domain_policy_copy(struct vm_domain_policy *dst,
 158     const struct vm_domain_policy *src)
 159 {
 160         seq_t seq;
 161         struct vm_domain_policy d;
 162
 163         for (;;) {
 164                 seq = seq_read(&src->seq);
 165                 d = *src;
 166                 if (seq_consistent(&src->seq, seq)) {
 167                         seq_write_begin(&dst->seq);
 168                         dst->p.domain = d.p.domain;
 169                         dst->p.policy = d.p.policy;
 170                         seq_write_end(&dst->seq);
 171                         return;
 172                 }
 173                 cpu_spinwait();
 174         }
 175 }
 176
 177 int
 178 vm_domain_policy_validate(const struct vm_domain_policy *vp)
 179 {
 180
 181         switch (vp->p.policy) {
 182         case VM_POLICY_NONE:
 183         case VM_POLICY_ROUND_ROBIN:
 184         case VM_POLICY_FIRST_TOUCH:
 185         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 186                 if (vp->p.domain == -1)
 187                         return (0);
 188                 return (-1);
 189         case VM_POLICY_FIXED_DOMAIN:
 190         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 191 #ifdef VM_NUMA_ALLOC
 192                 if (vp->p.domain >= 0 && vp->p.domain < vm_ndomains)
 193                         return (0);
 194 #else
 195                 if (vp->p.domain == 0)
 196                         return (0);
 197 #endif
 198                 return (-1);
 199         default:
 200                 return (-1);
 201         }
 202         return (-1);
 203 }
 204
 205 int
 206 vm_domain_policy_cleanup(struct vm_domain_policy *vp)
 207 {
 208
 209         /* For now, empty */
 210         return (0);
 211 }
 212
 213 int
 214 vm_domain_iterator_init(struct vm_domain_iterator *vi)
 215 {
 216
 217         /* Nothing to do for now */
 218         return (0);
 219 }
 220
 221 /*
 222  * Manually setup an iterator with the given details.
 223  */
 224 int
 225 vm_domain_iterator_set(struct vm_domain_iterator *vi,
 226     vm_domain_policy_type_t vt, int domain)
 227 {
 228
 229 #ifdef VM_NUMA_ALLOC
 230         switch (vt) {
 231         case VM_POLICY_FIXED_DOMAIN:
 232                 vi->policy = VM_POLICY_FIXED_DOMAIN;
 233                 vi->domain = domain;
 234                 vi->n = 1;
 235                 break;
 236         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 237                 vi->policy = VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN;
 238                 vi->domain = domain;
 239                 vi->n = vm_ndomains;
 240                 break;
 241         case VM_POLICY_FIRST_TOUCH:
 242                 vi->policy = VM_POLICY_FIRST_TOUCH;
 243                 vi->domain = PCPU_GET(domain);
 244                 vi->n = 1;
 245                 break;
 246         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 247                 vi->policy = VM_POLICY_FIRST_TOUCH_ROUND_ROBIN;
 248                 vi->domain = PCPU_GET(domain);
 249                 vi->n = vm_ndomains;
 250                 break;
 251         case VM_POLICY_ROUND_ROBIN:
 252         default:
 253                 vi->policy = VM_POLICY_ROUND_ROBIN;
 254                 vi->domain = -1;
 255                 vi->n = vm_ndomains;
 256                 break;
 257         }
 258 #else
 259         vi->domain = 0;
 260         vi->n = 1;
 261 #endif
 262         return (0);
 263 }
 264
 265 /*
 266  * Setup an iterator based on the given policy.
 267  */
 268 static inline void
 269 _vm_domain_iterator_set_policy(struct vm_domain_iterator *vi,
 270     const struct vm_domain_policy *vt)
 271 {
 272
 273 #ifdef VM_NUMA_ALLOC
 274         /*
 275          * Initialise the iterator.
 276          *
 277          * For first-touch, the initial domain is set
 278          * via the current thread CPU domain.
 279          *
 280          * For fixed-domain, it's assumed that the
 281          * caller has initialised the specific domain
 282          * it is after.
 283          */
 284         switch (vt->p.policy) {
 285         case VM_POLICY_FIXED_DOMAIN:
 286                 vi->policy = vt->p.policy;
 287                 vi->domain = vt->p.domain;
 288                 vi->n = 1;
 289                 break;
 290         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 291                 vi->policy = vt->p.policy;
 292                 vi->domain = vt->p.domain;
 293                 vi->n = vm_ndomains;
 294                 break;
 295         case VM_POLICY_FIRST_TOUCH:
 296                 vi->policy = vt->p.policy;
 297                 vi->domain = PCPU_GET(domain);
 298                 vi->n = 1;
 299                 break;
 300         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 301                 vi->policy = vt->p.policy;
 302                 vi->domain = PCPU_GET(domain);
 303                 vi->n = vm_ndomains;
 304                 break;
 305         case VM_POLICY_ROUND_ROBIN:
 306         default:
 307                 /*
 308                  * Default to round-robin policy.
 309                  */
 310                 vi->policy = VM_POLICY_ROUND_ROBIN;
 311                 vi->domain = -1;
 312                 vi->n = vm_ndomains;
 313                 break;
 314         }
 315 #else
 316         vi->domain = 0;
 317         vi->n = 1;
 318 #endif
 319 }
 320
 321 void
 322 vm_domain_iterator_set_policy(struct vm_domain_iterator *vi,
 323     const struct vm_domain_policy *vt)
 324 {
 325         seq_t seq;
 326         struct vm_domain_policy vt_lcl;
 327
 328         for (;;) {
 329                 seq = seq_read(&vt->seq);
 330                 vt_lcl = *vt;
 331                 if (seq_consistent(&vt->seq, seq)) {
 332                         _vm_domain_iterator_set_policy(vi, &vt_lcl);
 333                         return;
 334                 }
 335                 cpu_spinwait();
 336         }
 337 }
 338
 339 /*
 340  * Return the next VM domain to use.
 341  *
 342  * Returns 0 w/ domain set to the next domain to use, or
 343  * -1 to indicate no more domains are available.
 344  */
 345 int
 346 vm_domain_iterator_run(struct vm_domain_iterator *vi, int *domain)
 347 {
 348
 349         /* General catch-all */
 350         if (vi->n <= 0)
 351                 return (-1);
 352
 353 #ifdef VM_NUMA_ALLOC
 354         switch (vi->policy) {
 355         case VM_POLICY_FIXED_DOMAIN:
 356         case VM_POLICY_FIRST_TOUCH:
 357                 *domain = vi->domain;
 358                 vi->n--;
 359                 break;
 360         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 361         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 362                 /*
 363                  * XXX TODO: skip over the rr'ed domain
 364                  * if it equals the one we started with.
 365                  */
 366                 if (vi->n == vm_ndomains)
 367                         *domain = vi->domain;
 368                 else
 369                         *domain = vm_domain_rr_selectdomain(vi->domain);
 370                 vi->n--;
 371                 break;
 372         case VM_POLICY_ROUND_ROBIN:
 373         default:
 374                 *domain = vm_domain_rr_selectdomain(-1);
 375                 vi->n--;
 376                 break;
 377         }
 378 #else
 379         *domain = 0;
 380         vi->n--;
 381 #endif
 382
 383         return (0);
 384 }
 385
 386 /*
 387  * Returns 1 if the iteration is done, or 0 if it has not.
 388
 389  * This can only be called after at least one loop through
 390  * the iterator.  Ie, it's designed to be used as a tail
 391  * check of a loop, not the head check of a loop.
 392  */
 393 int
 394 vm_domain_iterator_isdone(struct vm_domain_iterator *vi)
 395 {
 396
 397         return (vi->n <= 0);
 398 }
 399
 400 int
 401 vm_domain_iterator_cleanup(struct vm_domain_iterator *vi)
 402 {
 403
 404         return (0);
 405 }