sys/vm/vm_domain.c

   1 /*-
   2  * Copyright (c) 2015 Adrian Chadd <adrian@FreeBSD.org>.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer,
  10  *    without modification.
  11  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  12  *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
  13  *    redistribution must be conditioned upon including a substantially
  14  *    similar Disclaimer requirement for further binary redistribution.
  15  *
  16  * NO WARRANTY
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19  * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
  20  * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
  22  * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  23  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  25  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  26  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  27  * THE POSSIBILITY OF SUCH DAMAGES.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include "opt_vm.h"
  34 #include "opt_ddb.h"
  35
  36 #include <sys/param.h>
  37 #include <sys/systm.h>
  38 #include <sys/lock.h>
  39 #include <sys/kernel.h>
  40 #include <sys/malloc.h>
  41 #include <sys/mutex.h>
  42 #if MAXMEMDOM > 1
  43 #include <sys/proc.h>
  44 #endif
  45 #include <sys/queue.h>
  46 #include <sys/rwlock.h>
  47 #include <sys/sbuf.h>
  48 #include <sys/sysctl.h>
  49 #include <sys/tree.h>
  50 #include <sys/vmmeter.h>
  51 #include <sys/seq.h>
  52
  53 #include <ddb/ddb.h>
  54
  55 #include <vm/vm.h>
  56 #include <vm/vm_param.h>
  57 #include <vm/vm_kern.h>
  58 #include <vm/vm_object.h>
  59 #include <vm/vm_page.h>
  60 #include <vm/vm_phys.h>
  61
  62 #include <vm/vm_domain.h>
  63
  64 static __inline int
  65 vm_domain_rr_selectdomain(int skip_domain)
  66 {
  67 #if MAXMEMDOM > 1
  68         struct thread *td;
  69
  70         td = curthread;
  71
  72         td->td_dom_rr_idx++;
  73         td->td_dom_rr_idx %= vm_ndomains;
  74
  75         /*
  76          * If skip_domain is provided then skip over that
  77          * domain.  This is intended for round robin variants
  78          * which first try a fixed domain.
  79          */
  80         if ((skip_domain > -1) && (td->td_dom_rr_idx == skip_domain)) {
  81                 td->td_dom_rr_idx++;
  82                 td->td_dom_rr_idx %= vm_ndomains;
  83         }
  84         return (td->td_dom_rr_idx);
  85 #else
  86         return (0);
  87 #endif
  88 }
  89
  90 /*
  91  * This implements a very simple set of VM domain memory allocation
  92  * policies and iterators.
  93  */
  94
  95 /*
  96  * A VM domain policy represents a desired VM domain policy.
  97  * Iterators implement searching through VM domains in a specific
  98  * order.
  99  */
 100
 101 /*
 102  * When setting a policy, the caller must establish their own
 103  * exclusive write protection for the contents of the domain
 104  * policy.
 105  */
 106 int
 107 vm_domain_policy_init(struct vm_domain_policy *vp)
 108 {
 109
 110         bzero(vp, sizeof(*vp));
 111         vp->p.policy = VM_POLICY_NONE;
 112         vp->p.domain = -1;
 113         return (0);
 114 }
 115
 116 int
 117 vm_domain_policy_set(struct vm_domain_policy *vp,
 118     vm_domain_policy_type_t vt, int domain)
 119 {
 120
 121         seq_write_begin(&vp->seq);
 122         vp->p.policy = vt;
 123         vp->p.domain = domain;
 124         seq_write_end(&vp->seq);
 125         return (0);
 126 }
 127
 128 /*
 129  * Take a local copy of a policy.
 130  *
 131  * The destination policy isn't write-barriered; this is used
 132  * for doing local copies into something that isn't shared.
 133  */
 134 void
 135 vm_domain_policy_localcopy(struct vm_domain_policy *dst,
 136     const struct vm_domain_policy *src)
 137 {
 138         seq_t seq;
 139
 140         for (;;) {
 141                 seq = seq_read(&src->seq);
 142                 *dst = *src;
 143                 if (seq_consistent(&src->seq, seq))
 144                         return;
 145                 cpu_spinwait();
 146         }
 147 }
 148
 149 /*
 150  * Take a write-barrier copy of a policy.
 151  *
 152  * The destination policy is write -barriered; this is used
 153  * for doing copies into policies that may be read by other
 154  * threads.
 155  */
 156 void
 157 vm_domain_policy_copy(struct vm_domain_policy *dst,
 158     const struct vm_domain_policy *src)
 159 {
 160         seq_t seq;
 161         struct vm_domain_policy d;
 162
 163         for (;;) {
 164                 seq = seq_read(&src->seq);
 165                 d = *src;
 166                 if (seq_consistent(&src->seq, seq)) {
 167                         seq_write_begin(&dst->seq);
 168                         dst->p.domain = d.p.domain;
 169                         dst->p.policy = d.p.policy;
 170                         seq_write_end(&dst->seq);
 171                         return;
 172                 }
 173                 cpu_spinwait();
 174         }
 175 }
 176
 177 int
 178 vm_domain_policy_validate(const struct vm_domain_policy *vp)
 179 {
 180
 181         switch (vp->p.policy) {
 182         case VM_POLICY_NONE:
 183         case VM_POLICY_ROUND_ROBIN:
 184         case VM_POLICY_FIRST_TOUCH:
 185         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 186                 if (vp->p.domain == -1)
 187                         return (0);
 188                 return (-1);
 189         case VM_POLICY_FIXED_DOMAIN:
 190         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 191                 if (vp->p.domain >= 0 && vp->p.domain < vm_ndomains)
 192                         return (0);
 193                 return (-1);
 194         default:
 195                 return (-1);
 196         }
 197         return (-1);
 198 }
 199
 200 int
 201 vm_domain_policy_cleanup(struct vm_domain_policy *vp)
 202 {
 203
 204         /* For now, empty */
 205         return (0);
 206 }
 207
 208 int
 209 vm_domain_iterator_init(struct vm_domain_iterator *vi)
 210 {
 211
 212         /* Nothing to do for now */
 213         return (0);
 214 }
 215
 216 /*
 217  * Manually setup an iterator with the given details.
 218  */
 219 int
 220 vm_domain_iterator_set(struct vm_domain_iterator *vi,
 221     vm_domain_policy_type_t vt, int domain)
 222 {
 223
 224         switch (vt) {
 225         case VM_POLICY_FIXED_DOMAIN:
 226                 vi->policy = VM_POLICY_FIXED_DOMAIN;
 227                 vi->domain = domain;
 228                 vi->n = 1;
 229                 break;
 230         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 231                 vi->policy = VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN;
 232                 vi->domain = domain;
 233                 vi->n = vm_ndomains;
 234                 break;
 235         case VM_POLICY_FIRST_TOUCH:
 236                 vi->policy = VM_POLICY_FIRST_TOUCH;
 237                 vi->domain = PCPU_GET(domain);
 238                 vi->n = 1;
 239                 break;
 240         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 241                 vi->policy = VM_POLICY_FIRST_TOUCH_ROUND_ROBIN;
 242                 vi->domain = PCPU_GET(domain);
 243                 vi->n = vm_ndomains;
 244                 break;
 245         case VM_POLICY_ROUND_ROBIN:
 246         default:
 247                 vi->policy = VM_POLICY_ROUND_ROBIN;
 248                 vi->domain = -1;
 249                 vi->n = vm_ndomains;
 250                 break;
 251         }
 252         return (0);
 253 }
 254
 255 /*
 256  * Setup an iterator based on the given policy.
 257  */
 258 static inline void
 259 _vm_domain_iterator_set_policy(struct vm_domain_iterator *vi,
 260     const struct vm_domain_policy *vt)
 261 {
 262         /*
 263          * Initialise the iterator.
 264          *
 265          * For first-touch, the initial domain is set
 266          * via the current thread CPU domain.
 267          *
 268          * For fixed-domain, it's assumed that the
 269          * caller has initialised the specific domain
 270          * it is after.
 271          */
 272         switch (vt->p.policy) {
 273         case VM_POLICY_FIXED_DOMAIN:
 274                 vi->policy = vt->p.policy;
 275                 vi->domain = vt->p.domain;
 276                 vi->n = 1;
 277                 break;
 278         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 279                 vi->policy = vt->p.policy;
 280                 vi->domain = vt->p.domain;
 281                 vi->n = vm_ndomains;
 282                 break;
 283         case VM_POLICY_FIRST_TOUCH:
 284                 vi->policy = vt->p.policy;
 285                 vi->domain = PCPU_GET(domain);
 286                 vi->n = 1;
 287                 break;
 288         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 289                 vi->policy = vt->p.policy;
 290                 vi->domain = PCPU_GET(domain);
 291                 vi->n = vm_ndomains;
 292                 break;
 293         case VM_POLICY_ROUND_ROBIN:
 294         default:
 295                 /*
 296                  * Default to round-robin policy.
 297                  */
 298                 vi->policy = VM_POLICY_ROUND_ROBIN;
 299                 vi->domain = -1;
 300                 vi->n = vm_ndomains;
 301                 break;
 302         }
 303 }
 304
 305 void
 306 vm_domain_iterator_set_policy(struct vm_domain_iterator *vi,
 307     const struct vm_domain_policy *vt)
 308 {
 309         seq_t seq;
 310         struct vm_domain_policy vt_lcl;
 311
 312         for (;;) {
 313                 seq = seq_read(&vt->seq);
 314                 vt_lcl = *vt;
 315                 if (seq_consistent(&vt->seq, seq)) {
 316                         _vm_domain_iterator_set_policy(vi, &vt_lcl);
 317                         return;
 318                 }
 319                 cpu_spinwait();
 320         }
 321 }
 322
 323 /*
 324  * Return the next VM domain to use.
 325  *
 326  * Returns 0 w/ domain set to the next domain to use, or
 327  * -1 to indicate no more domains are available.
 328  */
 329 int
 330 vm_domain_iterator_run(struct vm_domain_iterator *vi, int *domain)
 331 {
 332
 333         /* General catch-all */
 334         if (vi->n <= 0)
 335                 return (-1);
 336
 337         switch (vi->policy) {
 338         case VM_POLICY_FIXED_DOMAIN:
 339         case VM_POLICY_FIRST_TOUCH:
 340                 *domain = vi->domain;
 341                 vi->n--;
 342                 break;
 343         case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
 344         case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
 345                 /*
 346                  * XXX TODO: skip over the rr'ed domain
 347                  * if it equals the one we started with.
 348                  */
 349                 if (vi->n == vm_ndomains)
 350                         *domain = vi->domain;
 351                 else
 352                         *domain = vm_domain_rr_selectdomain(vi->domain);
 353                 vi->n--;
 354                 break;
 355         case VM_POLICY_ROUND_ROBIN:
 356         default:
 357                 *domain = vm_domain_rr_selectdomain(-1);
 358                 vi->n--;
 359                 break;
 360         }
 361
 362         return (0);
 363 }
 364
 365 /*
 366  * Returns 1 if the iteration is done, or 0 if it has not.
 367
 368  * This can only be called after at least one loop through
 369  * the iterator.  Ie, it's designed to be used as a tail
 370  * check of a loop, not the head check of a loop.
 371  */
 372 int
 373 vm_domain_iterator_isdone(struct vm_domain_iterator *vi)
 374 {
 375
 376         return (vi->n <= 0);
 377 }
 378
 379 int
 380 vm_domain_iterator_cleanup(struct vm_domain_iterator *vi)
 381 {
 382
 383         return (0);
 384 }