sys/vm/vm_domain.c

   1 /*-
   2  * Copyright (c) 2015 Adrian Chadd <adrian@FreeBSD.org>.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer,
  10  *    without modification.
  11  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  12  *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
  13  *    redistribution must be conditioned upon including a substantially
  14  *    similar Disclaimer requirement for further binary redistribution.
  15  *
  16  * NO WARRANTY
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19  * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
  20  * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
  22  * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  23  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  25  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  26  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  27  * THE POSSIBILITY OF SUCH DAMAGES.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include "opt_vm.h"
  34 #include "opt_ddb.h"
  35
  36 #include <sys/param.h>
  37 #include <sys/systm.h>
  38 #include <sys/lock.h>
  39 #include <sys/kernel.h>
  40 #include <sys/malloc.h>
  41 #include <sys/mutex.h>
  42 #ifdef VM_NUMA_ALLOC
  43 #include <sys/proc.h>
  44 #endif
  45 #include <sys/queue.h>
  46 #include <sys/rwlock.h>
  47 #include <sys/sbuf.h>
  48 #include <sys/sysctl.h>
  49 #include <sys/tree.h>
  50 #include <sys/vmmeter.h>
  51 #include <sys/seq.h>
  52
  53 #include <ddb/ddb.h>
  54
  55 #include <vm/vm.h>
  56 #include <vm/vm_param.h>
  57 #include <vm/vm_kern.h>
  58 #include <vm/vm_object.h>
  59 #include <vm/vm_page.h>
  60 #include <vm/vm_phys.h>
  61
  62 #include <vm/vm_domain.h>
  63
  64 #ifdef VM_NUMA_ALLOC
  65 static __inline int
  66 vm_domain_rr_selectdomain(int skip_domain)
  67 {
  68         struct thread *td;
  69
  70         td = curthread;
  71
  72         td->td_dom_rr_idx++;
  73         td->td_dom_rr_idx %= vm_ndomains;
  74
  75         /*
  76          * If skip_domain is provided then skip over that
  77          * domain.  This is intended for round robin variants
  78          * which first try a fixed domain.
  79          */
  80         if ((skip_domain > -1) && (td->td_dom_rr_idx == skip_domain)) {
  81                 td->td_dom_rr_idx++;
  82                 td->td_dom_rr_idx %= vm_ndomains;
  83         }
  84         return (td->td_dom_rr_idx);
  85 }
  86 #endif
  87
  88 /*
  89  * This implements a very simple set of VM domain memory allocation
  90  * policies and iterators.
  91  */
  92
  93 /*
  94  * A VM domain policy represents a desired VM domain policy.
  95  * Iterators implement searching through VM domains in a specific
  96  * order.
  97  */
  98
  99 /*
 100  * When setting a policy, the caller must establish their own
 101  * exclusive write protection for the contents of the domain
 102  * policy.
 103  */
 104 int
 105 vm_domain_policy_init(struct vm_domain_policy *vp)
 106 {
 107
 108         bzero(vp, sizeof(*vp));
 109         vp->p.policy = VM_POLICY_NONE;
 110         vp->p.domain = -1;
 111         return (0);
 112 }
 113
 114 int
 115 vm_domain_policy_set(struct vm_domain_policy *vp,
 116     vm_domain_policy_type_t vt, int domain)
 117 {
 118
 119         seq_write_begin(&vp->seq);
 120         vp->p.policy = vt;
 121         vp->p.domain = domain;
 122         seq_write_end(&vp->seq);
 123         return (0);
 124 }
 125
 126 /*
 127  * Take a local copy of a policy.
 128  *
 129  * The destination policy isn't write-barriered; this is used
 130  * for doing local copies into something that isn't shared.
 131  */
 132 void
 133 vm_domain_policy_localcopy(struct vm_domain_policy *dst,
 134     const struct vm_domain_policy *src)
 135 {
 136         seq_t seq;
 137
 138         for (;;) {
 139                 seq = seq_read(&src->seq);
 140                 *dst = *src;
 141                 if (seq_consistent(&src->seq, seq))
 142                         return;
 143                 cpu_spinwait();
 144         }
 145 }
 146
 147 /*
 148  * Take a write-barrier copy of a policy.
 149  *
 150  * The destination policy is write -barriered; this is used
 151  * for doing copies into policies that may be read by other
 152  * threads.
 153  */
 154 void
 155 vm_domain_policy_copy(struct vm_domain_policy *dst,
 156     const struct vm_domain_policy *src)
 157 {
 158         seq_t seq;
 159         struct vm_domain_policy d;
 160
 161         for (;;) {
 162                 seq = seq_read(&src->seq);
 163                 d = *src;
 164                 if (seq_consistent(&src->seq, seq)) {
 165                         seq_write_begin(&dst->seq);
 166                         dst->p.domain = d.p.domain;
 167                         dst->p.policy = d.p.policy;
 168                         seq_write_end(&dst->seq);
 169                         return;
 170                 }
 171                 cpu_spinwait();
 172         }
 173 }
 174
 175 int
 176 vm_domain_policy_validate(const struct vm_domain_policy *vp)
 177 {
 178
 179         switch (vp->p.policy) {
 180         case VM_POLICY_NONE:
 181         case VM_POLICY_ROUND_ROBIN:
 182         case VM_POLICY_FIRST_TOUCH:
 183         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 184                 if (vp->p.domain == -1)
 185                         return (0);
 186                 return (-1);
 187         case VM_POLICY_FIXED_DOMAIN:
 188         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 189 #ifdef VM_NUMA_ALLOC
 190                 if (vp->p.domain >= 0 && vp->p.domain < vm_ndomains)
 191                         return (0);
 192 #else
 193                 if (vp->p.domain == 0)
 194                         return (0);
 195 #endif
 196                 return (-1);
 197         default:
 198                 return (-1);
 199         }
 200         return (-1);
 201 }
 202
 203 int
 204 vm_domain_policy_cleanup(struct vm_domain_policy *vp)
 205 {
 206
 207         /* For now, empty */
 208         return (0);
 209 }
 210
 211 int
 212 vm_domain_iterator_init(struct vm_domain_iterator *vi)
 213 {
 214
 215         /* Nothing to do for now */
 216         return (0);
 217 }
 218
 219 /*
 220  * Manually setup an iterator with the given details.
 221  */
 222 int
 223 vm_domain_iterator_set(struct vm_domain_iterator *vi,
 224     vm_domain_policy_type_t vt, int domain)
 225 {
 226
 227 #ifdef VM_NUMA_ALLOC
 228         switch (vt) {
 229         case VM_POLICY_FIXED_DOMAIN:
 230                 vi->policy = VM_POLICY_FIXED_DOMAIN;
 231                 vi->domain = domain;
 232                 vi->n = 1;
 233                 break;
 234         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 235                 vi->policy = VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN;
 236                 vi->domain = domain;
 237                 vi->n = vm_ndomains;
 238                 break;
 239         case VM_POLICY_FIRST_TOUCH:
 240                 vi->policy = VM_POLICY_FIRST_TOUCH;
 241                 vi->domain = PCPU_GET(domain);
 242                 vi->n = 1;
 243                 break;
 244         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 245                 vi->policy = VM_POLICY_FIRST_TOUCH_ROUND_ROBIN;
 246                 vi->domain = PCPU_GET(domain);
 247                 vi->n = vm_ndomains;
 248                 break;
 249         case VM_POLICY_ROUND_ROBIN:
 250         default:
 251                 vi->policy = VM_POLICY_ROUND_ROBIN;
 252                 vi->domain = -1;
 253                 vi->n = vm_ndomains;
 254                 break;
 255         }
 256 #else
 257         vi->domain = 0;
 258         vi->n = 1;
 259 #endif
 260         return (0);
 261 }
 262
 263 /*
 264  * Setup an iterator based on the given policy.
 265  */
 266 static inline void
 267 _vm_domain_iterator_set_policy(struct vm_domain_iterator *vi,
 268     const struct vm_domain_policy *vt)
 269 {
 270
 271 #ifdef VM_NUMA_ALLOC
 272         /*
 273          * Initialise the iterator.
 274          *
 275          * For first-touch, the initial domain is set
 276          * via the current thread CPU domain.
 277          *
 278          * For fixed-domain, it's assumed that the
 279          * caller has initialised the specific domain
 280          * it is after.
 281          */
 282         switch (vt->p.policy) {
 283         case VM_POLICY_FIXED_DOMAIN:
 284                 vi->policy = vt->p.policy;
 285                 vi->domain = vt->p.domain;
 286                 vi->n = 1;
 287                 break;
 288         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 289                 vi->policy = vt->p.policy;
 290                 vi->domain = vt->p.domain;
 291                 vi->n = vm_ndomains;
 292                 break;
 293         case VM_POLICY_FIRST_TOUCH:
 294                 vi->policy = vt->p.policy;
 295                 vi->domain = PCPU_GET(domain);
 296                 vi->n = 1;
 297                 break;
 298         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 299                 vi->policy = vt->p.policy;
 300                 vi->domain = PCPU_GET(domain);
 301                 vi->n = vm_ndomains;
 302                 break;
 303         case VM_POLICY_ROUND_ROBIN:
 304         default:
 305                 /*
 306                  * Default to round-robin policy.
 307                  */
 308                 vi->policy = VM_POLICY_ROUND_ROBIN;
 309                 vi->domain = -1;
 310                 vi->n = vm_ndomains;
 311                 break;
 312         }
 313 #else
 314         vi->domain = 0;
 315         vi->n = 1;
 316 #endif
 317 }
 318
 319 void
 320 vm_domain_iterator_set_policy(struct vm_domain_iterator *vi,
 321     const struct vm_domain_policy *vt)
 322 {
 323         seq_t seq;
 324         struct vm_domain_policy vt_lcl;
 325
 326         for (;;) {
 327                 seq = seq_read(&vt->seq);
 328                 vt_lcl = *vt;
 329                 if (seq_consistent(&vt->seq, seq)) {
 330                         _vm_domain_iterator_set_policy(vi, &vt_lcl);
 331                         return;
 332                 }
 333                 cpu_spinwait();
 334         }
 335 }
 336
 337 /*
 338  * Return the next VM domain to use.
 339  *
 340  * Returns 0 w/ domain set to the next domain to use, or
 341  * -1 to indicate no more domains are available.
 342  */
 343 int
 344 vm_domain_iterator_run(struct vm_domain_iterator *vi, int *domain)
 345 {
 346
 347         /* General catch-all */
 348         if (vi->n <= 0)
 349                 return (-1);
 350
 351 #ifdef VM_NUMA_ALLOC
 352         switch (vi->policy) {
 353         case VM_POLICY_FIXED_DOMAIN:
 354         case VM_POLICY_FIRST_TOUCH:
 355                 *domain = vi->domain;
 356                 vi->n--;
 357                 break;
 358         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 359         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 360                 /*
 361                  * XXX TODO: skip over the rr'ed domain
 362                  * if it equals the one we started with.
 363                  */
 364                 if (vi->n == vm_ndomains)
 365                         *domain = vi->domain;
 366                 else
 367                         *domain = vm_domain_rr_selectdomain(vi->domain);
 368                 vi->n--;
 369                 break;
 370         case VM_POLICY_ROUND_ROBIN:
 371         default:
 372                 *domain = vm_domain_rr_selectdomain(-1);
 373                 vi->n--;
 374                 break;
 375         }
 376 #else
 377         *domain = 0;
 378         vi->n--;
 379 #endif
 380
 381         return (0);
 382 }
 383
 384 /*
 385  * Returns 1 if the iteration is done, or 0 if it has not.
 386
 387  * This can only be called after at least one loop through
 388  * the iterator.  Ie, it's designed to be used as a tail
 389  * check of a loop, not the head check of a loop.
 390  */
 391 int
 392 vm_domain_iterator_isdone(struct vm_domain_iterator *vi)
 393 {
 394
 395         return (vi->n <= 0);
 396 }
 397
 398 int
 399 vm_domain_iterator_cleanup(struct vm_domain_iterator *vi)
 400 {
 401
 402         return (0);
 403 }