sys/kern/kern_umtx.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2015, 2016 The FreeBSD Foundation
   5  * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
   6  * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
   7  * All rights reserved.
   8  *
   9  * Portions of this software were developed by Konstantin Belousov
  10  * under sponsorship from the FreeBSD Foundation.
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice unmodified, this list of conditions, and the following
  17  *    disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  23  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  24  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  25  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  27  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  31  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include "opt_umtx_profiling.h"
  38
  39 #include <sys/param.h>
  40 #include <sys/kernel.h>
  41 #include <sys/fcntl.h>
  42 #include <sys/file.h>
  43 #include <sys/filedesc.h>
  44 #include <sys/limits.h>
  45 #include <sys/lock.h>
  46 #include <sys/malloc.h>
  47 #include <sys/mman.h>
  48 #include <sys/mutex.h>
  49 #include <sys/priv.h>
  50 #include <sys/proc.h>
  51 #include <sys/resource.h>
  52 #include <sys/resourcevar.h>
  53 #include <sys/rwlock.h>
  54 #include <sys/sbuf.h>
  55 #include <sys/sched.h>
  56 #include <sys/smp.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/sysent.h>
  59 #include <sys/systm.h>
  60 #include <sys/sysproto.h>
  61 #include <sys/syscallsubr.h>
  62 #include <sys/taskqueue.h>
  63 #include <sys/time.h>
  64 #include <sys/eventhandler.h>
  65 #include <sys/umtx.h>
  66
  67 #include <security/mac/mac_framework.h>
  68
  69 #include <vm/vm.h>
  70 #include <vm/vm_param.h>
  71 #include <vm/pmap.h>
  72 #include <vm/vm_map.h>
  73 #include <vm/vm_object.h>
  74
  75 #include <machine/atomic.h>
  76 #include <machine/cpu.h>
  77
  78 #ifdef COMPAT_FREEBSD32
  79 #include <compat/freebsd32/freebsd32_proto.h>
  80 #endif
  81
  82 #define _UMUTEX_TRY             1
  83 #define _UMUTEX_WAIT            2
  84
  85 #ifdef UMTX_PROFILING
  86 #define UPROF_PERC_BIGGER(w, f, sw, sf)                                 \
  87         (((w) > (sw)) || ((w) == (sw) && (f) > (sf)))
  88 #endif
  89
  90 /* Priority inheritance mutex info. */
  91 struct umtx_pi {
  92         /* Owner thread */
  93         struct thread           *pi_owner;
  94
  95         /* Reference count */
  96         int                     pi_refcount;
  97
  98         /* List entry to link umtx holding by thread */
  99         TAILQ_ENTRY(umtx_pi)    pi_link;
 100
 101         /* List entry in hash */
 102         TAILQ_ENTRY(umtx_pi)    pi_hashlink;
 103
 104         /* List for waiters */
 105         TAILQ_HEAD(,umtx_q)     pi_blocked;
 106
 107         /* Identify a userland lock object */
 108         struct umtx_key         pi_key;
 109 };
 110
 111 /* A userland synchronous object user. */
 112 struct umtx_q {
 113         /* Linked list for the hash. */
 114         TAILQ_ENTRY(umtx_q)     uq_link;
 115
 116         /* Umtx key. */
 117         struct umtx_key         uq_key;
 118
 119         /* Umtx flags. */
 120         int                     uq_flags;
 121 #define UQF_UMTXQ       0x0001
 122
 123         /* The thread waits on. */
 124         struct thread           *uq_thread;
 125
 126         /*
 127          * Blocked on PI mutex. read can use chain lock
 128          * or umtx_lock, write must have both chain lock and
 129          * umtx_lock being hold.
 130          */
 131         struct umtx_pi          *uq_pi_blocked;
 132
 133         /* On blocked list */
 134         TAILQ_ENTRY(umtx_q)     uq_lockq;
 135
 136         /* Thread contending with us */
 137         TAILQ_HEAD(,umtx_pi)    uq_pi_contested;
 138
 139         /* Inherited priority from PP mutex */
 140         u_char                  uq_inherited_pri;
 141
 142         /* Spare queue ready to be reused */
 143         struct umtxq_queue      *uq_spare_queue;
 144
 145         /* The queue we on */
 146         struct umtxq_queue      *uq_cur_queue;
 147 };
 148
 149 TAILQ_HEAD(umtxq_head, umtx_q);
 150
 151 /* Per-key wait-queue */
 152 struct umtxq_queue {
 153         struct umtxq_head       head;
 154         struct umtx_key         key;
 155         LIST_ENTRY(umtxq_queue) link;
 156         int                     length;
 157 };
 158
 159 LIST_HEAD(umtxq_list, umtxq_queue);
 160
 161 /* Userland lock object's wait-queue chain */
 162 struct umtxq_chain {
 163         /* Lock for this chain. */
 164         struct mtx              uc_lock;
 165
 166         /* List of sleep queues. */
 167         struct umtxq_list       uc_queue[2];
 168 #define UMTX_SHARED_QUEUE       0
 169 #define UMTX_EXCLUSIVE_QUEUE    1
 170
 171         LIST_HEAD(, umtxq_queue) uc_spare_queue;
 172
 173         /* Busy flag */
 174         char                    uc_busy;
 175
 176         /* Chain lock waiters */
 177         int                     uc_waiters;
 178
 179         /* All PI in the list */
 180         TAILQ_HEAD(,umtx_pi)    uc_pi_list;
 181
 182 #ifdef UMTX_PROFILING
 183         u_int                   length;
 184         u_int                   max_length;
 185 #endif
 186 };
 187
 188 #define UMTXQ_LOCKED_ASSERT(uc)         mtx_assert(&(uc)->uc_lock, MA_OWNED)
 189
 190 /*
 191  * Don't propagate time-sharing priority, there is a security reason,
 192  * a user can simply introduce PI-mutex, let thread A lock the mutex,
 193  * and let another thread B block on the mutex, because B is
 194  * sleeping, its priority will be boosted, this causes A's priority to
 195  * be boosted via priority propagating too and will never be lowered even
 196  * if it is using 100%CPU, this is unfair to other processes.
 197  */
 198
 199 #define UPRI(td)        (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\
 200                           (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\
 201                          PRI_MAX_TIMESHARE : (td)->td_user_pri)
 202
 203 #define GOLDEN_RATIO_PRIME      2654404609U
 204 #ifndef UMTX_CHAINS
 205 #define UMTX_CHAINS             512
 206 #endif
 207 #define UMTX_SHIFTS             (__WORD_BIT - 9)
 208
 209 #define GET_SHARE(flags)        \
 210     (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE)
 211
 212 #define BUSY_SPINS              200
 213
 214 struct abs_timeout {
 215         int clockid;
 216         bool is_abs_real;       /* TIMER_ABSTIME && CLOCK_REALTIME* */
 217         struct timespec cur;
 218         struct timespec end;
 219 };
 220
 221 #ifdef COMPAT_FREEBSD32
 222 struct umutex32 {
 223         volatile __lwpid_t      m_owner;        /* Owner of the mutex */
 224         __uint32_t              m_flags;        /* Flags of the mutex */
 225         __uint32_t              m_ceilings[2];  /* Priority protect ceiling */
 226         __uint32_t              m_rb_lnk;       /* Robust linkage */
 227         __uint32_t              m_pad;
 228         __uint32_t              m_spare[2];
 229 };
 230
 231 _Static_assert(sizeof(struct umutex) == sizeof(struct umutex32), "umutex32");
 232 _Static_assert(__offsetof(struct umutex, m_spare[0]) ==
 233     __offsetof(struct umutex32, m_spare[0]), "m_spare32");
 234 #endif
 235
 236 int umtx_shm_vnobj_persistent = 0;
 237 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_vnode_persistent, CTLFLAG_RWTUN,
 238     &umtx_shm_vnobj_persistent, 0,
 239     "False forces destruction of umtx attached to file, on last close");
 240 static int umtx_max_rb = 1000;
 241 SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_max_robust, CTLFLAG_RWTUN,
 242     &umtx_max_rb, 0,
 243     "Maximum number of robust mutexes allowed for each thread");
 244
 245 static uma_zone_t               umtx_pi_zone;
 246 static struct umtxq_chain       umtxq_chains[2][UMTX_CHAINS];
 247 static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory");
 248 static int                      umtx_pi_allocated;
 249
 250 static SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 251     "umtx debug");
 252 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD,
 253     &umtx_pi_allocated, 0, "Allocated umtx_pi");
 254 static int umtx_verbose_rb = 1;
 255 SYSCTL_INT(_debug_umtx, OID_AUTO, robust_faults_verbose, CTLFLAG_RWTUN,
 256     &umtx_verbose_rb, 0,
 257     "");
 258
 259 #ifdef UMTX_PROFILING
 260 static long max_length;
 261 SYSCTL_LONG(_debug_umtx, OID_AUTO, max_length, CTLFLAG_RD, &max_length, 0, "max_length");
 262 static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 263     "umtx chain stats");
 264 #endif
 265
 266 static void abs_timeout_update(struct abs_timeout *timo);
 267
 268 static void umtx_shm_init(void);
 269 static void umtxq_sysinit(void *);
 270 static void umtxq_hash(struct umtx_key *key);
 271 static struct umtxq_chain *umtxq_getchain(struct umtx_key *key);
 272 static void umtxq_lock(struct umtx_key *key);
 273 static void umtxq_unlock(struct umtx_key *key);
 274 static void umtxq_busy(struct umtx_key *key);
 275 static void umtxq_unbusy(struct umtx_key *key);
 276 static void umtxq_insert_queue(struct umtx_q *uq, int q);
 277 static void umtxq_remove_queue(struct umtx_q *uq, int q);
 278 static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *);
 279 static int umtxq_count(struct umtx_key *key);
 280 static struct umtx_pi *umtx_pi_alloc(int);
 281 static void umtx_pi_free(struct umtx_pi *pi);
 282 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags,
 283     bool rb);
 284 static void umtx_thread_cleanup(struct thread *td);
 285 static void umtx_exec_hook(void *arg __unused, struct proc *p __unused,
 286     struct image_params *imgp __unused);
 287 SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL);
 288
 289 #define umtxq_signal(key, nwake)        umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE)
 290 #define umtxq_insert(uq)        umtxq_insert_queue((uq), UMTX_SHARED_QUEUE)
 291 #define umtxq_remove(uq)        umtxq_remove_queue((uq), UMTX_SHARED_QUEUE)
 292
 293 static struct mtx umtx_lock;
 294
 295 #ifdef UMTX_PROFILING
 296 static void
 297 umtx_init_profiling(void)
 298 {
 299         struct sysctl_oid *chain_oid;
 300         char chain_name[10];
 301         int i;
 302
 303         for (i = 0; i < UMTX_CHAINS; ++i) {
 304                 snprintf(chain_name, sizeof(chain_name), "%d", i);
 305                 chain_oid = SYSCTL_ADD_NODE(NULL,
 306                     SYSCTL_STATIC_CHILDREN(_debug_umtx_chains), OID_AUTO,
 307                     chain_name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
 308                     "umtx hash stats");
 309                 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
 310                     "max_length0", CTLFLAG_RD, &umtxq_chains[0][i].max_length, 0, NULL);
 311                 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
 312                     "max_length1", CTLFLAG_RD, &umtxq_chains[1][i].max_length, 0, NULL);
 313         }
 314 }
 315
 316 static int
 317 sysctl_debug_umtx_chains_peaks(SYSCTL_HANDLER_ARGS)
 318 {
 319         char buf[512];
 320         struct sbuf sb;
 321         struct umtxq_chain *uc;
 322         u_int fract, i, j, tot, whole;
 323         u_int sf0, sf1, sf2, sf3, sf4;
 324         u_int si0, si1, si2, si3, si4;
 325         u_int sw0, sw1, sw2, sw3, sw4;
 326
 327         sbuf_new(&sb, buf, sizeof(buf), SBUF_FIXEDLEN);
 328         for (i = 0; i < 2; i++) {
 329                 tot = 0;
 330                 for (j = 0; j < UMTX_CHAINS; ++j) {
 331                         uc = &umtxq_chains[i][j];
 332                         mtx_lock(&uc->uc_lock);
 333                         tot += uc->max_length;
 334                         mtx_unlock(&uc->uc_lock);
 335                 }
 336                 if (tot == 0)
 337                         sbuf_printf(&sb, "%u) Empty ", i);
 338                 else {
 339                         sf0 = sf1 = sf2 = sf3 = sf4 = 0;
 340                         si0 = si1 = si2 = si3 = si4 = 0;
 341                         sw0 = sw1 = sw2 = sw3 = sw4 = 0;
 342                         for (j = 0; j < UMTX_CHAINS; j++) {
 343                                 uc = &umtxq_chains[i][j];
 344                                 mtx_lock(&uc->uc_lock);
 345                                 whole = uc->max_length * 100;
 346                                 mtx_unlock(&uc->uc_lock);
 347                                 fract = (whole % tot) * 100;
 348                                 if (UPROF_PERC_BIGGER(whole, fract, sw0, sf0)) {
 349                                         sf0 = fract;
 350                                         si0 = j;
 351                                         sw0 = whole;
 352                                 } else if (UPROF_PERC_BIGGER(whole, fract, sw1,
 353                                     sf1)) {
 354                                         sf1 = fract;
 355                                         si1 = j;
 356                                         sw1 = whole;
 357                                 } else if (UPROF_PERC_BIGGER(whole, fract, sw2,
 358                                     sf2)) {
 359                                         sf2 = fract;
 360                                         si2 = j;
 361                                         sw2 = whole;
 362                                 } else if (UPROF_PERC_BIGGER(whole, fract, sw3,
 363                                     sf3)) {
 364                                         sf3 = fract;
 365                                         si3 = j;
 366                                         sw3 = whole;
 367                                 } else if (UPROF_PERC_BIGGER(whole, fract, sw4,
 368                                     sf4)) {
 369                                         sf4 = fract;
 370                                         si4 = j;
 371                                         sw4 = whole;
 372                                 }
 373                         }
 374                         sbuf_printf(&sb, "queue %u:\n", i);
 375                         sbuf_printf(&sb, "1st: %u.%u%% idx: %u\n", sw0 / tot,
 376                             sf0 / tot, si0);
 377                         sbuf_printf(&sb, "2nd: %u.%u%% idx: %u\n", sw1 / tot,
 378                             sf1 / tot, si1);
 379                         sbuf_printf(&sb, "3rd: %u.%u%% idx: %u\n", sw2 / tot,
 380                             sf2 / tot, si2);
 381                         sbuf_printf(&sb, "4th: %u.%u%% idx: %u\n", sw3 / tot,
 382                             sf3 / tot, si3);
 383                         sbuf_printf(&sb, "5th: %u.%u%% idx: %u\n", sw4 / tot,
 384                             sf4 / tot, si4);
 385                 }
 386         }
 387         sbuf_trim(&sb);
 388         sbuf_finish(&sb);
 389         sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
 390         sbuf_delete(&sb);
 391         return (0);
 392 }
 393
 394 static int
 395 sysctl_debug_umtx_chains_clear(SYSCTL_HANDLER_ARGS)
 396 {
 397         struct umtxq_chain *uc;
 398         u_int i, j;
 399         int clear, error;
 400
 401         clear = 0;
 402         error = sysctl_handle_int(oidp, &clear, 0, req);
 403         if (error != 0 || req->newptr == NULL)
 404                 return (error);
 405
 406         if (clear != 0) {
 407                 for (i = 0; i < 2; ++i) {
 408                         for (j = 0; j < UMTX_CHAINS; ++j) {
 409                                 uc = &umtxq_chains[i][j];
 410                                 mtx_lock(&uc->uc_lock);
 411                                 uc->length = 0;
 412                                 uc->max_length = 0;
 413                                 mtx_unlock(&uc->uc_lock);
 414                         }
 415                 }
 416         }
 417         return (0);
 418 }
 419
 420 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, clear,
 421     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0,
 422     sysctl_debug_umtx_chains_clear, "I",
 423     "Clear umtx chains statistics");
 424 SYSCTL_PROC(_debug_umtx_chains, OID_AUTO, peaks,
 425     CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 0,
 426     sysctl_debug_umtx_chains_peaks, "A",
 427     "Highest peaks in chains max length");
 428 #endif
 429
 430 static void
 431 umtxq_sysinit(void *arg __unused)
 432 {
 433         int i, j;
 434
 435         umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi),
 436                 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 437         for (i = 0; i < 2; ++i) {
 438                 for (j = 0; j < UMTX_CHAINS; ++j) {
 439                         mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL,
 440                                  MTX_DEF | MTX_DUPOK);
 441                         LIST_INIT(&umtxq_chains[i][j].uc_queue[0]);
 442                         LIST_INIT(&umtxq_chains[i][j].uc_queue[1]);
 443                         LIST_INIT(&umtxq_chains[i][j].uc_spare_queue);
 444                         TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list);
 445                         umtxq_chains[i][j].uc_busy = 0;
 446                         umtxq_chains[i][j].uc_waiters = 0;
 447 #ifdef UMTX_PROFILING
 448                         umtxq_chains[i][j].length = 0;
 449                         umtxq_chains[i][j].max_length = 0;
 450 #endif
 451                 }
 452         }
 453 #ifdef UMTX_PROFILING
 454         umtx_init_profiling();
 455 #endif
 456         mtx_init(&umtx_lock, "umtx lock", NULL, MTX_DEF);
 457         EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL,
 458             EVENTHANDLER_PRI_ANY);
 459         umtx_shm_init();
 460 }
 461
 462 struct umtx_q *
 463 umtxq_alloc(void)
 464 {
 465         struct umtx_q *uq;
 466
 467         uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO);
 468         uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX,
 469             M_WAITOK | M_ZERO);
 470         TAILQ_INIT(&uq->uq_spare_queue->head);
 471         TAILQ_INIT(&uq->uq_pi_contested);
 472         uq->uq_inherited_pri = PRI_MAX;
 473         return (uq);
 474 }
 475
 476 void
 477 umtxq_free(struct umtx_q *uq)
 478 {
 479
 480         MPASS(uq->uq_spare_queue != NULL);
 481         free(uq->uq_spare_queue, M_UMTX);
 482         free(uq, M_UMTX);
 483 }
 484
 485 static inline void
 486 umtxq_hash(struct umtx_key *key)
 487 {
 488         unsigned n;
 489
 490         n = (uintptr_t)key->info.both.a + key->info.both.b;
 491         key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS;
 492 }
 493
 494 static inline struct umtxq_chain *
 495 umtxq_getchain(struct umtx_key *key)
 496 {
 497
 498         if (key->type <= TYPE_SEM)
 499                 return (&umtxq_chains[1][key->hash]);
 500         return (&umtxq_chains[0][key->hash]);
 501 }
 502
 503 /*
 504  * Lock a chain.
 505  */
 506 static inline void
 507 umtxq_lock(struct umtx_key *key)
 508 {
 509         struct umtxq_chain *uc;
 510
 511         uc = umtxq_getchain(key);
 512         mtx_lock(&uc->uc_lock);
 513 }
 514
 515 /*
 516  * Unlock a chain.
 517  */
 518 static inline void
 519 umtxq_unlock(struct umtx_key *key)
 520 {
 521         struct umtxq_chain *uc;
 522
 523         uc = umtxq_getchain(key);
 524         mtx_unlock(&uc->uc_lock);
 525 }
 526
 527 /*
 528  * Set chain to busy state when following operation
 529  * may be blocked (kernel mutex can not be used).
 530  */
 531 static inline void
 532 umtxq_busy(struct umtx_key *key)
 533 {
 534         struct umtxq_chain *uc;
 535
 536         uc = umtxq_getchain(key);
 537         mtx_assert(&uc->uc_lock, MA_OWNED);
 538         if (uc->uc_busy) {
 539 #ifdef SMP
 540                 if (smp_cpus > 1) {
 541                         int count = BUSY_SPINS;
 542                         if (count > 0) {
 543                                 umtxq_unlock(key);
 544                                 while (uc->uc_busy && --count > 0)
 545                                         cpu_spinwait();
 546                                 umtxq_lock(key);
 547                         }
 548                 }
 549 #endif
 550                 while (uc->uc_busy) {
 551                         uc->uc_waiters++;
 552                         msleep(uc, &uc->uc_lock, 0, "umtxqb", 0);
 553                         uc->uc_waiters--;
 554                 }
 555         }
 556         uc->uc_busy = 1;
 557 }
 558
 559 /*
 560  * Unbusy a chain.
 561  */
 562 static inline void
 563 umtxq_unbusy(struct umtx_key *key)
 564 {
 565         struct umtxq_chain *uc;
 566
 567         uc = umtxq_getchain(key);
 568         mtx_assert(&uc->uc_lock, MA_OWNED);
 569         KASSERT(uc->uc_busy != 0, ("not busy"));
 570         uc->uc_busy = 0;
 571         if (uc->uc_waiters)
 572                 wakeup_one(uc);
 573 }
 574
 575 static inline void
 576 umtxq_unbusy_unlocked(struct umtx_key *key)
 577 {
 578
 579         umtxq_lock(key);
 580         umtxq_unbusy(key);
 581         umtxq_unlock(key);
 582 }
 583
 584 static struct umtxq_queue *
 585 umtxq_queue_lookup(struct umtx_key *key, int q)
 586 {
 587         struct umtxq_queue *uh;
 588         struct umtxq_chain *uc;
 589
 590         uc = umtxq_getchain(key);
 591         UMTXQ_LOCKED_ASSERT(uc);
 592         LIST_FOREACH(uh, &uc->uc_queue[q], link) {
 593                 if (umtx_key_match(&uh->key, key))
 594                         return (uh);
 595         }
 596
 597         return (NULL);
 598 }
 599
 600 static inline void
 601 umtxq_insert_queue(struct umtx_q *uq, int q)
 602 {
 603         struct umtxq_queue *uh;
 604         struct umtxq_chain *uc;
 605
 606         uc = umtxq_getchain(&uq->uq_key);
 607         UMTXQ_LOCKED_ASSERT(uc);
 608         KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue"));
 609         uh = umtxq_queue_lookup(&uq->uq_key, q);
 610         if (uh != NULL) {
 611                 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link);
 612         } else {
 613                 uh = uq->uq_spare_queue;
 614                 uh->key = uq->uq_key;
 615                 LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link);
 616 #ifdef UMTX_PROFILING
 617                 uc->length++;
 618                 if (uc->length > uc->max_length) {
 619                         uc->max_length = uc->length;
 620                         if (uc->max_length > max_length)
 621                                 max_length = uc->max_length;
 622                 }
 623 #endif
 624         }
 625         uq->uq_spare_queue = NULL;
 626
 627         TAILQ_INSERT_TAIL(&uh->head, uq, uq_link);
 628         uh->length++;
 629         uq->uq_flags |= UQF_UMTXQ;
 630         uq->uq_cur_queue = uh;
 631         return;
 632 }
 633
 634 static inline void
 635 umtxq_remove_queue(struct umtx_q *uq, int q)
 636 {
 637         struct umtxq_chain *uc;
 638         struct umtxq_queue *uh;
 639
 640         uc = umtxq_getchain(&uq->uq_key);
 641         UMTXQ_LOCKED_ASSERT(uc);
 642         if (uq->uq_flags & UQF_UMTXQ) {
 643                 uh = uq->uq_cur_queue;
 644                 TAILQ_REMOVE(&uh->head, uq, uq_link);
 645                 uh->length--;
 646                 uq->uq_flags &= ~UQF_UMTXQ;
 647                 if (TAILQ_EMPTY(&uh->head)) {
 648                         KASSERT(uh->length == 0,
 649                             ("inconsistent umtxq_queue length"));
 650 #ifdef UMTX_PROFILING
 651                         uc->length--;
 652 #endif
 653                         LIST_REMOVE(uh, link);
 654                 } else {
 655                         uh = LIST_FIRST(&uc->uc_spare_queue);
 656                         KASSERT(uh != NULL, ("uc_spare_queue is empty"));
 657                         LIST_REMOVE(uh, link);
 658                 }
 659                 uq->uq_spare_queue = uh;
 660                 uq->uq_cur_queue = NULL;
 661         }
 662 }
 663
 664 /*
 665  * Check if there are multiple waiters
 666  */
 667 static int
 668 umtxq_count(struct umtx_key *key)
 669 {
 670         struct umtxq_queue *uh;
 671
 672         UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
 673         uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
 674         if (uh != NULL)
 675                 return (uh->length);
 676         return (0);
 677 }
 678
 679 /*
 680  * Check if there are multiple PI waiters and returns first
 681  * waiter.
 682  */
 683 static int
 684 umtxq_count_pi(struct umtx_key *key, struct umtx_q **first)
 685 {
 686         struct umtxq_queue *uh;
 687
 688         *first = NULL;
 689         UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
 690         uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
 691         if (uh != NULL) {
 692                 *first = TAILQ_FIRST(&uh->head);
 693                 return (uh->length);
 694         }
 695         return (0);
 696 }
 697
 698 /*
 699  * Wake up threads waiting on an userland object.
 700  */
 701
 702 static int
 703 umtxq_signal_queue(struct umtx_key *key, int n_wake, int q)
 704 {
 705         struct umtxq_queue *uh;
 706         struct umtx_q *uq;
 707         int ret;
 708
 709         ret = 0;
 710         UMTXQ_LOCKED_ASSERT(umtxq_getchain(key));
 711         uh = umtxq_queue_lookup(key, q);
 712         if (uh != NULL) {
 713                 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) {
 714                         umtxq_remove_queue(uq, q);
 715                         wakeup(uq);
 716                         if (++ret >= n_wake)
 717                                 return (ret);
 718                 }
 719         }
 720         return (ret);
 721 }
 722
 723 /*
 724  * Wake up specified thread.
 725  */
 726 static inline void
 727 umtxq_signal_thread(struct umtx_q *uq)
 728 {
 729
 730         UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
 731         umtxq_remove(uq);
 732         wakeup(uq);
 733 }
 734
 735 static inline int
 736 tstohz(const struct timespec *tsp)
 737 {
 738         struct timeval tv;
 739
 740         TIMESPEC_TO_TIMEVAL(&tv, tsp);
 741         return tvtohz(&tv);
 742 }
 743
 744 static void
 745 abs_timeout_init(struct abs_timeout *timo, int clockid, int absolute,
 746         const struct timespec *timeout)
 747 {
 748
 749         timo->clockid = clockid;
 750         if (!absolute) {
 751                 timo->is_abs_real = false;
 752                 abs_timeout_update(timo);
 753                 timespecadd(&timo->cur, timeout, &timo->end);
 754         } else {
 755                 timo->end = *timeout;
 756                 timo->is_abs_real = clockid == CLOCK_REALTIME ||
 757                     clockid == CLOCK_REALTIME_FAST ||
 758                     clockid == CLOCK_REALTIME_PRECISE;
 759                 /*
 760                  * If is_abs_real, umtxq_sleep will read the clock
 761                  * after setting td_rtcgen; otherwise, read it here.
 762                  */
 763                 if (!timo->is_abs_real) {
 764                         abs_timeout_update(timo);
 765                 }
 766         }
 767 }
 768
 769 static void
 770 abs_timeout_init2(struct abs_timeout *timo, const struct _umtx_time *umtxtime)
 771 {
 772
 773         abs_timeout_init(timo, umtxtime->_clockid,
 774             (umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout);
 775 }
 776
 777 static inline void
 778 abs_timeout_update(struct abs_timeout *timo)
 779 {
 780
 781         kern_clock_gettime(curthread, timo->clockid, &timo->cur);
 782 }
 783
 784 static int
 785 abs_timeout_gethz(struct abs_timeout *timo)
 786 {
 787         struct timespec tts;
 788
 789         if (timespeccmp(&timo->end, &timo->cur, <=))
 790                 return (-1);
 791         timespecsub(&timo->end, &timo->cur, &tts);
 792         return (tstohz(&tts));
 793 }
 794
 795 static uint32_t
 796 umtx_unlock_val(uint32_t flags, bool rb)
 797 {
 798
 799         if (rb)
 800                 return (UMUTEX_RB_OWNERDEAD);
 801         else if ((flags & UMUTEX_NONCONSISTENT) != 0)
 802                 return (UMUTEX_RB_NOTRECOV);
 803         else
 804                 return (UMUTEX_UNOWNED);
 805
 806 }
 807
 808 /*
 809  * Put thread into sleep state, before sleeping, check if
 810  * thread was removed from umtx queue.
 811  */
 812 static inline int
 813 umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *abstime)
 814 {
 815         struct umtxq_chain *uc;
 816         int error, timo;
 817
 818         if (abstime != NULL && abstime->is_abs_real) {
 819                 curthread->td_rtcgen = atomic_load_acq_int(&rtc_generation);
 820                 abs_timeout_update(abstime);
 821         }
 822
 823         uc = umtxq_getchain(&uq->uq_key);
 824         UMTXQ_LOCKED_ASSERT(uc);
 825         for (;;) {
 826                 if (!(uq->uq_flags & UQF_UMTXQ)) {
 827                         error = 0;
 828                         break;
 829                 }
 830                 if (abstime != NULL) {
 831                         timo = abs_timeout_gethz(abstime);
 832                         if (timo < 0) {
 833                                 error = ETIMEDOUT;
 834                                 break;
 835                         }
 836                 } else
 837                         timo = 0;
 838                 error = msleep(uq, &uc->uc_lock, PCATCH | PDROP, wmesg, timo);
 839                 if (error == EINTR || error == ERESTART) {
 840                         umtxq_lock(&uq->uq_key);
 841                         break;
 842                 }
 843                 if (abstime != NULL) {
 844                         if (abstime->is_abs_real)
 845                                 curthread->td_rtcgen =
 846                                     atomic_load_acq_int(&rtc_generation);
 847                         abs_timeout_update(abstime);
 848                 }
 849                 umtxq_lock(&uq->uq_key);
 850         }
 851
 852         curthread->td_rtcgen = 0;
 853         return (error);
 854 }
 855
 856 /*
 857  * Convert userspace address into unique logical address.
 858  */
 859 int
 860 umtx_key_get(const void *addr, int type, int share, struct umtx_key *key)
 861 {
 862         struct thread *td = curthread;
 863         vm_map_t map;
 864         vm_map_entry_t entry;
 865         vm_pindex_t pindex;
 866         vm_prot_t prot;
 867         boolean_t wired;
 868
 869         key->type = type;
 870         if (share == THREAD_SHARE) {
 871                 key->shared = 0;
 872                 key->info.private.vs = td->td_proc->p_vmspace;
 873                 key->info.private.addr = (uintptr_t)addr;
 874         } else {
 875                 MPASS(share == PROCESS_SHARE || share == AUTO_SHARE);
 876                 map = &td->td_proc->p_vmspace->vm_map;
 877                 if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE,
 878                     &entry, &key->info.shared.object, &pindex, &prot,
 879                     &wired) != KERN_SUCCESS) {
 880                         return (EFAULT);
 881                 }
 882
 883                 if ((share == PROCESS_SHARE) ||
 884                     (share == AUTO_SHARE &&
 885                      VM_INHERIT_SHARE == entry->inheritance)) {
 886                         key->shared = 1;
 887                         key->info.shared.offset = (vm_offset_t)addr -
 888                             entry->start + entry->offset;
 889                         vm_object_reference(key->info.shared.object);
 890                 } else {
 891                         key->shared = 0;
 892                         key->info.private.vs = td->td_proc->p_vmspace;
 893                         key->info.private.addr = (uintptr_t)addr;
 894                 }
 895                 vm_map_lookup_done(map, entry);
 896         }
 897
 898         umtxq_hash(key);
 899         return (0);
 900 }
 901
 902 /*
 903  * Release key.
 904  */
 905 void
 906 umtx_key_release(struct umtx_key *key)
 907 {
 908         if (key->shared)
 909                 vm_object_deallocate(key->info.shared.object);
 910 }
 911
 912 /*
 913  * Fetch and compare value, sleep on the address if value is not changed.
 914  */
 915 static int
 916 do_wait(struct thread *td, void *addr, u_long id,
 917     struct _umtx_time *timeout, int compat32, int is_private)
 918 {
 919         struct abs_timeout timo;
 920         struct umtx_q *uq;
 921         u_long tmp;
 922         uint32_t tmp32;
 923         int error = 0;
 924
 925         uq = td->td_umtxq;
 926         if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT,
 927                 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0)
 928                 return (error);
 929
 930         if (timeout != NULL)
 931                 abs_timeout_init2(&timo, timeout);
 932
 933         umtxq_lock(&uq->uq_key);
 934         umtxq_insert(uq);
 935         umtxq_unlock(&uq->uq_key);
 936         if (compat32 == 0) {
 937                 error = fueword(addr, &tmp);
 938                 if (error != 0)
 939                         error = EFAULT;
 940         } else {
 941                 error = fueword32(addr, &tmp32);
 942                 if (error == 0)
 943                         tmp = tmp32;
 944                 else
 945                         error = EFAULT;
 946         }
 947         umtxq_lock(&uq->uq_key);
 948         if (error == 0) {
 949                 if (tmp == id)
 950                         error = umtxq_sleep(uq, "uwait", timeout == NULL ?
 951                             NULL : &timo);
 952                 if ((uq->uq_flags & UQF_UMTXQ) == 0)
 953                         error = 0;
 954                 else
 955                         umtxq_remove(uq);
 956         } else if ((uq->uq_flags & UQF_UMTXQ) != 0) {
 957                 umtxq_remove(uq);
 958         }
 959         umtxq_unlock(&uq->uq_key);
 960         umtx_key_release(&uq->uq_key);
 961         if (error == ERESTART)
 962                 error = EINTR;
 963         return (error);
 964 }
 965
 966 /*
 967  * Wake up threads sleeping on the specified address.
 968  */
 969 int
 970 kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private)
 971 {
 972         struct umtx_key key;
 973         int ret;
 974
 975         if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT,
 976             is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0)
 977                 return (ret);
 978         umtxq_lock(&key);
 979         umtxq_signal(&key, n_wake);
 980         umtxq_unlock(&key);
 981         umtx_key_release(&key);
 982         return (0);
 983 }
 984
 985 /*
 986  * Lock PTHREAD_PRIO_NONE protocol POSIX mutex.
 987  */
 988 static int
 989 do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags,
 990     struct _umtx_time *timeout, int mode)
 991 {
 992         struct abs_timeout timo;
 993         struct umtx_q *uq;
 994         uint32_t owner, old, id;
 995         int error, rv;
 996
 997         id = td->td_tid;
 998         uq = td->td_umtxq;
 999         error = 0;
1000         if (timeout != NULL)
1001                 abs_timeout_init2(&timo, timeout);
1002
1003         /*
1004          * Care must be exercised when dealing with umtx structure. It
1005          * can fault on any access.
1006          */
1007         for (;;) {
1008                 rv = fueword32(&m->m_owner, &owner);
1009                 if (rv == -1)
1010                         return (EFAULT);
1011                 if (mode == _UMUTEX_WAIT) {
1012                         if (owner == UMUTEX_UNOWNED ||
1013                             owner == UMUTEX_CONTESTED ||
1014                             owner == UMUTEX_RB_OWNERDEAD ||
1015                             owner == UMUTEX_RB_NOTRECOV)
1016                                 return (0);
1017                 } else {
1018                         /*
1019                          * Robust mutex terminated.  Kernel duty is to
1020                          * return EOWNERDEAD to the userspace.  The
1021                          * umutex.m_flags UMUTEX_NONCONSISTENT is set
1022                          * by the common userspace code.
1023                          */
1024                         if (owner == UMUTEX_RB_OWNERDEAD) {
1025                                 rv = casueword32(&m->m_owner,
1026                                     UMUTEX_RB_OWNERDEAD, &owner,
1027                                     id | UMUTEX_CONTESTED);
1028                                 if (rv == -1)
1029                                         return (EFAULT);
1030                                 if (rv == 0) {
1031                                         MPASS(owner == UMUTEX_RB_OWNERDEAD);
1032                                         return (EOWNERDEAD); /* success */
1033                                 }
1034                                 MPASS(rv == 1);
1035                                 rv = thread_check_susp(td, false);
1036                                 if (rv != 0)
1037                                         return (rv);
1038                                 continue;
1039                         }
1040                         if (owner == UMUTEX_RB_NOTRECOV)
1041                                 return (ENOTRECOVERABLE);
1042
1043                         /*
1044                          * Try the uncontested case.  This should be
1045                          * done in userland.
1046                          */
1047                         rv = casueword32(&m->m_owner, UMUTEX_UNOWNED,
1048                             &owner, id);
1049                         /* The address was invalid. */
1050                         if (rv == -1)
1051                                 return (EFAULT);
1052
1053                         /* The acquire succeeded. */
1054                         if (rv == 0) {
1055                                 MPASS(owner == UMUTEX_UNOWNED);
1056                                 return (0);
1057                         }
1058
1059                         /*
1060                          * If no one owns it but it is contested try
1061                          * to acquire it.
1062                          */
1063                         MPASS(rv == 1);
1064                         if (owner == UMUTEX_CONTESTED) {
1065                                 rv = casueword32(&m->m_owner,
1066                                     UMUTEX_CONTESTED, &owner,
1067                                     id | UMUTEX_CONTESTED);
1068                                 /* The address was invalid. */
1069                                 if (rv == -1)
1070                                         return (EFAULT);
1071                                 if (rv == 0) {
1072                                         MPASS(owner == UMUTEX_CONTESTED);
1073                                         return (0);
1074                                 }
1075                                 if (rv == 1) {
1076                                         rv = thread_check_susp(td, false);
1077                                         if (rv != 0)
1078                                                 return (rv);
1079                                 }
1080
1081                                 /*
1082                                  * If this failed the lock has
1083                                  * changed, restart.
1084                                  */
1085                                 continue;
1086                         }
1087
1088                         /* rv == 1 but not contested, likely store failure */
1089                         rv = thread_check_susp(td, false);
1090                         if (rv != 0)
1091                                 return (rv);
1092                 }
1093
1094                 if (mode == _UMUTEX_TRY)
1095                         return (EBUSY);
1096
1097                 /*
1098                  * If we caught a signal, we have retried and now
1099                  * exit immediately.
1100                  */
1101                 if (error != 0)
1102                         return (error);
1103
1104                 if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX,
1105                     GET_SHARE(flags), &uq->uq_key)) != 0)
1106                         return (error);
1107
1108                 umtxq_lock(&uq->uq_key);
1109                 umtxq_busy(&uq->uq_key);
1110                 umtxq_insert(uq);
1111                 umtxq_unlock(&uq->uq_key);
1112
1113                 /*
1114                  * Set the contested bit so that a release in user space
1115                  * knows to use the system call for unlock.  If this fails
1116                  * either some one else has acquired the lock or it has been
1117                  * released.
1118                  */
1119                 rv = casueword32(&m->m_owner, owner, &old,
1120                     owner | UMUTEX_CONTESTED);
1121
1122                 /* The address was invalid or casueword failed to store. */
1123                 if (rv == -1 || rv == 1) {
1124                         umtxq_lock(&uq->uq_key);
1125                         umtxq_remove(uq);
1126                         umtxq_unbusy(&uq->uq_key);
1127                         umtxq_unlock(&uq->uq_key);
1128                         umtx_key_release(&uq->uq_key);
1129                         if (rv == -1)
1130                                 return (EFAULT);
1131                         if (rv == 1) {
1132                                 rv = thread_check_susp(td, false);
1133                                 if (rv != 0)
1134                                         return (rv);
1135                         }
1136                         continue;
1137                 }
1138
1139                 /*
1140                  * We set the contested bit, sleep. Otherwise the lock changed
1141                  * and we need to retry or we lost a race to the thread
1142                  * unlocking the umtx.
1143                  */
1144                 umtxq_lock(&uq->uq_key);
1145                 umtxq_unbusy(&uq->uq_key);
1146                 MPASS(old == owner);
1147                 error = umtxq_sleep(uq, "umtxn", timeout == NULL ?
1148                     NULL : &timo);
1149                 umtxq_remove(uq);
1150                 umtxq_unlock(&uq->uq_key);
1151                 umtx_key_release(&uq->uq_key);
1152
1153                 if (error == 0)
1154                         error = thread_check_susp(td, false);
1155         }
1156
1157         return (0);
1158 }
1159
1160 /*
1161  * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex.
1162  */
1163 static int
1164 do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
1165 {
1166         struct umtx_key key;
1167         uint32_t owner, old, id, newlock;
1168         int error, count;
1169
1170         id = td->td_tid;
1171
1172 again:
1173         /*
1174          * Make sure we own this mtx.
1175          */
1176         error = fueword32(&m->m_owner, &owner);
1177         if (error == -1)
1178                 return (EFAULT);
1179
1180         if ((owner & ~UMUTEX_CONTESTED) != id)
1181                 return (EPERM);
1182
1183         newlock = umtx_unlock_val(flags, rb);
1184         if ((owner & UMUTEX_CONTESTED) == 0) {
1185                 error = casueword32(&m->m_owner, owner, &old, newlock);
1186                 if (error == -1)
1187                         return (EFAULT);
1188                 if (error == 1) {
1189                         error = thread_check_susp(td, false);
1190                         if (error != 0)
1191                                 return (error);
1192                         goto again;
1193                 }
1194                 MPASS(old == owner);
1195                 return (0);
1196         }
1197
1198         /* We should only ever be in here for contested locks */
1199         if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1200             &key)) != 0)
1201                 return (error);
1202
1203         umtxq_lock(&key);
1204         umtxq_busy(&key);
1205         count = umtxq_count(&key);
1206         umtxq_unlock(&key);
1207
1208         /*
1209          * When unlocking the umtx, it must be marked as unowned if
1210          * there is zero or one thread only waiting for it.
1211          * Otherwise, it must be marked as contested.
1212          */
1213         if (count > 1)
1214                 newlock |= UMUTEX_CONTESTED;
1215         error = casueword32(&m->m_owner, owner, &old, newlock);
1216         umtxq_lock(&key);
1217         umtxq_signal(&key, 1);
1218         umtxq_unbusy(&key);
1219         umtxq_unlock(&key);
1220         umtx_key_release(&key);
1221         if (error == -1)
1222                 return (EFAULT);
1223         if (error == 1) {
1224                 if (old != owner)
1225                         return (EINVAL);
1226                 error = thread_check_susp(td, false);
1227                 if (error != 0)
1228                         return (error);
1229                 goto again;
1230         }
1231         return (0);
1232 }
1233
1234 /*
1235  * Check if the mutex is available and wake up a waiter,
1236  * only for simple mutex.
1237  */
1238 static int
1239 do_wake_umutex(struct thread *td, struct umutex *m)
1240 {
1241         struct umtx_key key;
1242         uint32_t owner;
1243         uint32_t flags;
1244         int error;
1245         int count;
1246
1247 again:
1248         error = fueword32(&m->m_owner, &owner);
1249         if (error == -1)
1250                 return (EFAULT);
1251
1252         if ((owner & ~UMUTEX_CONTESTED) != 0 && owner != UMUTEX_RB_OWNERDEAD &&
1253             owner != UMUTEX_RB_NOTRECOV)
1254                 return (0);
1255
1256         error = fueword32(&m->m_flags, &flags);
1257         if (error == -1)
1258                 return (EFAULT);
1259
1260         /* We should only ever be in here for contested locks */
1261         if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
1262             &key)) != 0)
1263                 return (error);
1264
1265         umtxq_lock(&key);
1266         umtxq_busy(&key);
1267         count = umtxq_count(&key);
1268         umtxq_unlock(&key);
1269
1270         if (count <= 1 && owner != UMUTEX_RB_OWNERDEAD &&
1271             owner != UMUTEX_RB_NOTRECOV) {
1272                 error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
1273                     UMUTEX_UNOWNED);
1274                 if (error == -1) {
1275                         error = EFAULT;
1276                 } else if (error == 1) {
1277                         umtxq_lock(&key);
1278                         umtxq_unbusy(&key);
1279                         umtxq_unlock(&key);
1280                         umtx_key_release(&key);
1281                         error = thread_check_susp(td, false);
1282                         if (error != 0)
1283                                 return (error);
1284                         goto again;
1285                 }
1286         }
1287
1288         umtxq_lock(&key);
1289         if (error == 0 && count != 0) {
1290                 MPASS((owner & ~UMUTEX_CONTESTED) == 0 ||
1291                     owner == UMUTEX_RB_OWNERDEAD ||
1292                     owner == UMUTEX_RB_NOTRECOV);
1293                 umtxq_signal(&key, 1);
1294         }
1295         umtxq_unbusy(&key);
1296         umtxq_unlock(&key);
1297         umtx_key_release(&key);
1298         return (error);
1299 }
1300
1301 /*
1302  * Check if the mutex has waiters and tries to fix contention bit.
1303  */
1304 static int
1305 do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags)
1306 {
1307         struct umtx_key key;
1308         uint32_t owner, old;
1309         int type;
1310         int error;
1311         int count;
1312
1313         switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT |
1314             UMUTEX_ROBUST)) {
1315         case 0:
1316         case UMUTEX_ROBUST:
1317                 type = TYPE_NORMAL_UMUTEX;
1318                 break;
1319         case UMUTEX_PRIO_INHERIT:
1320                 type = TYPE_PI_UMUTEX;
1321                 break;
1322         case (UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST):
1323                 type = TYPE_PI_ROBUST_UMUTEX;
1324                 break;
1325         case UMUTEX_PRIO_PROTECT:
1326                 type = TYPE_PP_UMUTEX;
1327                 break;
1328         case (UMUTEX_PRIO_PROTECT | UMUTEX_ROBUST):
1329                 type = TYPE_PP_ROBUST_UMUTEX;
1330                 break;
1331         default:
1332                 return (EINVAL);
1333         }
1334         if ((error = umtx_key_get(m, type, GET_SHARE(flags), &key)) != 0)
1335                 return (error);
1336
1337         owner = 0;
1338         umtxq_lock(&key);
1339         umtxq_busy(&key);
1340         count = umtxq_count(&key);
1341         umtxq_unlock(&key);
1342
1343         error = fueword32(&m->m_owner, &owner);
1344         if (error == -1)
1345                 error = EFAULT;
1346
1347         /*
1348          * Only repair contention bit if there is a waiter, this means
1349          * the mutex is still being referenced by userland code,
1350          * otherwise don't update any memory.
1351          */
1352         while (error == 0 && (owner & UMUTEX_CONTESTED) == 0 &&
1353             (count > 1 || (count == 1 && (owner & ~UMUTEX_CONTESTED) != 0))) {
1354                 error = casueword32(&m->m_owner, owner, &old,
1355                     owner | UMUTEX_CONTESTED);
1356                 if (error == -1) {
1357                         error = EFAULT;
1358                         break;
1359                 }
1360                 if (error == 0) {
1361                         MPASS(old == owner);
1362                         break;
1363                 }
1364                 owner = old;
1365                 error = thread_check_susp(td, false);
1366         }
1367
1368         umtxq_lock(&key);
1369         if (error == EFAULT) {
1370                 umtxq_signal(&key, INT_MAX);
1371         } else if (count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 ||
1372             owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV))
1373                 umtxq_signal(&key, 1);
1374         umtxq_unbusy(&key);
1375         umtxq_unlock(&key);
1376         umtx_key_release(&key);
1377         return (error);
1378 }
1379
1380 static inline struct umtx_pi *
1381 umtx_pi_alloc(int flags)
1382 {
1383         struct umtx_pi *pi;
1384
1385         pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags);
1386         TAILQ_INIT(&pi->pi_blocked);
1387         atomic_add_int(&umtx_pi_allocated, 1);
1388         return (pi);
1389 }
1390
1391 static inline void
1392 umtx_pi_free(struct umtx_pi *pi)
1393 {
1394         uma_zfree(umtx_pi_zone, pi);
1395         atomic_add_int(&umtx_pi_allocated, -1);
1396 }
1397
1398 /*
1399  * Adjust the thread's position on a pi_state after its priority has been
1400  * changed.
1401  */
1402 static int
1403 umtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td)
1404 {
1405         struct umtx_q *uq, *uq1, *uq2;
1406         struct thread *td1;
1407
1408         mtx_assert(&umtx_lock, MA_OWNED);
1409         if (pi == NULL)
1410                 return (0);
1411
1412         uq = td->td_umtxq;
1413
1414         /*
1415          * Check if the thread needs to be moved on the blocked chain.
1416          * It needs to be moved if either its priority is lower than
1417          * the previous thread or higher than the next thread.
1418          */
1419         uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq);
1420         uq2 = TAILQ_NEXT(uq, uq_lockq);
1421         if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) ||
1422             (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) {
1423                 /*
1424                  * Remove thread from blocked chain and determine where
1425                  * it should be moved to.
1426                  */
1427                 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1428                 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1429                         td1 = uq1->uq_thread;
1430                         MPASS(td1->td_proc->p_magic == P_MAGIC);
1431                         if (UPRI(td1) > UPRI(td))
1432                                 break;
1433                 }
1434
1435                 if (uq1 == NULL)
1436                         TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1437                 else
1438                         TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1439         }
1440         return (1);
1441 }
1442
1443 static struct umtx_pi *
1444 umtx_pi_next(struct umtx_pi *pi)
1445 {
1446         struct umtx_q *uq_owner;
1447
1448         if (pi->pi_owner == NULL)
1449                 return (NULL);
1450         uq_owner = pi->pi_owner->td_umtxq;
1451         if (uq_owner == NULL)
1452                 return (NULL);
1453         return (uq_owner->uq_pi_blocked);
1454 }
1455
1456 /*
1457  * Floyd's Cycle-Finding Algorithm.
1458  */
1459 static bool
1460 umtx_pi_check_loop(struct umtx_pi *pi)
1461 {
1462         struct umtx_pi *pi1;    /* fast iterator */
1463
1464         mtx_assert(&umtx_lock, MA_OWNED);
1465         if (pi == NULL)
1466                 return (false);
1467         pi1 = pi;
1468         for (;;) {
1469                 pi = umtx_pi_next(pi);
1470                 if (pi == NULL)
1471                         break;
1472                 pi1 = umtx_pi_next(pi1);
1473                 if (pi1 == NULL)
1474                         break;
1475                 pi1 = umtx_pi_next(pi1);
1476                 if (pi1 == NULL)
1477                         break;
1478                 if (pi == pi1)
1479                         return (true);
1480         }
1481         return (false);
1482 }
1483
1484 /*
1485  * Propagate priority when a thread is blocked on POSIX
1486  * PI mutex.
1487  */
1488 static void
1489 umtx_propagate_priority(struct thread *td)
1490 {
1491         struct umtx_q *uq;
1492         struct umtx_pi *pi;
1493         int pri;
1494
1495         mtx_assert(&umtx_lock, MA_OWNED);
1496         pri = UPRI(td);
1497         uq = td->td_umtxq;
1498         pi = uq->uq_pi_blocked;
1499         if (pi == NULL)
1500                 return;
1501         if (umtx_pi_check_loop(pi))
1502                 return;
1503
1504         for (;;) {
1505                 td = pi->pi_owner;
1506                 if (td == NULL || td == curthread)
1507                         return;
1508
1509                 MPASS(td->td_proc != NULL);
1510                 MPASS(td->td_proc->p_magic == P_MAGIC);
1511
1512                 thread_lock(td);
1513                 if (td->td_lend_user_pri > pri)
1514                         sched_lend_user_prio(td, pri);
1515                 else {
1516                         thread_unlock(td);
1517                         break;
1518                 }
1519                 thread_unlock(td);
1520
1521                 /*
1522                  * Pick up the lock that td is blocked on.
1523                  */
1524                 uq = td->td_umtxq;
1525                 pi = uq->uq_pi_blocked;
1526                 if (pi == NULL)
1527                         break;
1528                 /* Resort td on the list if needed. */
1529                 umtx_pi_adjust_thread(pi, td);
1530         }
1531 }
1532
1533 /*
1534  * Unpropagate priority for a PI mutex when a thread blocked on
1535  * it is interrupted by signal or resumed by others.
1536  */
1537 static void
1538 umtx_repropagate_priority(struct umtx_pi *pi)
1539 {
1540         struct umtx_q *uq, *uq_owner;
1541         struct umtx_pi *pi2;
1542         int pri;
1543
1544         mtx_assert(&umtx_lock, MA_OWNED);
1545
1546         if (umtx_pi_check_loop(pi))
1547                 return;
1548         while (pi != NULL && pi->pi_owner != NULL) {
1549                 pri = PRI_MAX;
1550                 uq_owner = pi->pi_owner->td_umtxq;
1551
1552                 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) {
1553                         uq = TAILQ_FIRST(&pi2->pi_blocked);
1554                         if (uq != NULL) {
1555                                 if (pri > UPRI(uq->uq_thread))
1556                                         pri = UPRI(uq->uq_thread);
1557                         }
1558                 }
1559
1560                 if (pri > uq_owner->uq_inherited_pri)
1561                         pri = uq_owner->uq_inherited_pri;
1562                 thread_lock(pi->pi_owner);
1563                 sched_lend_user_prio(pi->pi_owner, pri);
1564                 thread_unlock(pi->pi_owner);
1565                 if ((pi = uq_owner->uq_pi_blocked) != NULL)
1566                         umtx_pi_adjust_thread(pi, uq_owner->uq_thread);
1567         }
1568 }
1569
1570 /*
1571  * Insert a PI mutex into owned list.
1572  */
1573 static void
1574 umtx_pi_setowner(struct umtx_pi *pi, struct thread *owner)
1575 {
1576         struct umtx_q *uq_owner;
1577
1578         uq_owner = owner->td_umtxq;
1579         mtx_assert(&umtx_lock, MA_OWNED);
1580         MPASS(pi->pi_owner == NULL);
1581         pi->pi_owner = owner;
1582         TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link);
1583 }
1584
1585 /*
1586  * Disown a PI mutex, and remove it from the owned list.
1587  */
1588 static void
1589 umtx_pi_disown(struct umtx_pi *pi)
1590 {
1591
1592         mtx_assert(&umtx_lock, MA_OWNED);
1593         TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, pi, pi_link);
1594         pi->pi_owner = NULL;
1595 }
1596
1597 /*
1598  * Claim ownership of a PI mutex.
1599  */
1600 static int
1601 umtx_pi_claim(struct umtx_pi *pi, struct thread *owner)
1602 {
1603         struct umtx_q *uq;
1604         int pri;
1605
1606         mtx_lock(&umtx_lock);
1607         if (pi->pi_owner == owner) {
1608                 mtx_unlock(&umtx_lock);
1609                 return (0);
1610         }
1611
1612         if (pi->pi_owner != NULL) {
1613                 /*
1614                  * userland may have already messed the mutex, sigh.
1615                  */
1616                 mtx_unlock(&umtx_lock);
1617                 return (EPERM);
1618         }
1619         umtx_pi_setowner(pi, owner);
1620         uq = TAILQ_FIRST(&pi->pi_blocked);
1621         if (uq != NULL) {
1622                 pri = UPRI(uq->uq_thread);
1623                 thread_lock(owner);
1624                 if (pri < UPRI(owner))
1625                         sched_lend_user_prio(owner, pri);
1626                 thread_unlock(owner);
1627         }
1628         mtx_unlock(&umtx_lock);
1629         return (0);
1630 }
1631
1632 /*
1633  * Adjust a thread's order position in its blocked PI mutex,
1634  * this may result new priority propagating process.
1635  */
1636 void
1637 umtx_pi_adjust(struct thread *td, u_char oldpri)
1638 {
1639         struct umtx_q *uq;
1640         struct umtx_pi *pi;
1641
1642         uq = td->td_umtxq;
1643         mtx_lock(&umtx_lock);
1644         /*
1645          * Pick up the lock that td is blocked on.
1646          */
1647         pi = uq->uq_pi_blocked;
1648         if (pi != NULL) {
1649                 umtx_pi_adjust_thread(pi, td);
1650                 umtx_repropagate_priority(pi);
1651         }
1652         mtx_unlock(&umtx_lock);
1653 }
1654
1655 /*
1656  * Sleep on a PI mutex.
1657  */
1658 static int
1659 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner,
1660     const char *wmesg, struct abs_timeout *timo, bool shared)
1661 {
1662         struct thread *td, *td1;
1663         struct umtx_q *uq1;
1664         int error, pri;
1665 #ifdef INVARIANTS
1666         struct umtxq_chain *uc;
1667
1668         uc = umtxq_getchain(&pi->pi_key);
1669 #endif
1670         error = 0;
1671         td = uq->uq_thread;
1672         KASSERT(td == curthread, ("inconsistent uq_thread"));
1673         UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
1674         KASSERT(uc->uc_busy != 0, ("umtx chain is not busy"));
1675         umtxq_insert(uq);
1676         mtx_lock(&umtx_lock);
1677         if (pi->pi_owner == NULL) {
1678                 mtx_unlock(&umtx_lock);
1679                 td1 = tdfind(owner, shared ? -1 : td->td_proc->p_pid);
1680                 mtx_lock(&umtx_lock);
1681                 if (td1 != NULL) {
1682                         if (pi->pi_owner == NULL)
1683                                 umtx_pi_setowner(pi, td1);
1684                         PROC_UNLOCK(td1->td_proc);
1685                 }
1686         }
1687
1688         TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1689                 pri = UPRI(uq1->uq_thread);
1690                 if (pri > UPRI(td))
1691                         break;
1692         }
1693
1694         if (uq1 != NULL)
1695                 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1696         else
1697                 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1698
1699         uq->uq_pi_blocked = pi;
1700         thread_lock(td);
1701         td->td_flags |= TDF_UPIBLOCKED;
1702         thread_unlock(td);
1703         umtx_propagate_priority(td);
1704         mtx_unlock(&umtx_lock);
1705         umtxq_unbusy(&uq->uq_key);
1706
1707         error = umtxq_sleep(uq, wmesg, timo);
1708         umtxq_remove(uq);
1709
1710         mtx_lock(&umtx_lock);
1711         uq->uq_pi_blocked = NULL;
1712         thread_lock(td);
1713         td->td_flags &= ~TDF_UPIBLOCKED;
1714         thread_unlock(td);
1715         TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1716         umtx_repropagate_priority(pi);
1717         mtx_unlock(&umtx_lock);
1718         umtxq_unlock(&uq->uq_key);
1719
1720         return (error);
1721 }
1722
1723 /*
1724  * Add reference count for a PI mutex.
1725  */
1726 static void
1727 umtx_pi_ref(struct umtx_pi *pi)
1728 {
1729
1730         UMTXQ_LOCKED_ASSERT(umtxq_getchain(&pi->pi_key));
1731         pi->pi_refcount++;
1732 }
1733
1734 /*
1735  * Decrease reference count for a PI mutex, if the counter
1736  * is decreased to zero, its memory space is freed.
1737  */
1738 static void
1739 umtx_pi_unref(struct umtx_pi *pi)
1740 {
1741         struct umtxq_chain *uc;
1742
1743         uc = umtxq_getchain(&pi->pi_key);
1744         UMTXQ_LOCKED_ASSERT(uc);
1745         KASSERT(pi->pi_refcount > 0, ("invalid reference count"));
1746         if (--pi->pi_refcount == 0) {
1747                 mtx_lock(&umtx_lock);
1748                 if (pi->pi_owner != NULL)
1749                         umtx_pi_disown(pi);
1750                 KASSERT(TAILQ_EMPTY(&pi->pi_blocked),
1751                         ("blocked queue not empty"));
1752                 mtx_unlock(&umtx_lock);
1753                 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink);
1754                 umtx_pi_free(pi);
1755         }
1756 }
1757
1758 /*
1759  * Find a PI mutex in hash table.
1760  */
1761 static struct umtx_pi *
1762 umtx_pi_lookup(struct umtx_key *key)
1763 {
1764         struct umtxq_chain *uc;
1765         struct umtx_pi *pi;
1766
1767         uc = umtxq_getchain(key);
1768         UMTXQ_LOCKED_ASSERT(uc);
1769
1770         TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) {
1771                 if (umtx_key_match(&pi->pi_key, key)) {
1772                         return (pi);
1773                 }
1774         }
1775         return (NULL);
1776 }
1777
1778 /*
1779  * Insert a PI mutex into hash table.
1780  */
1781 static inline void
1782 umtx_pi_insert(struct umtx_pi *pi)
1783 {
1784         struct umtxq_chain *uc;
1785
1786         uc = umtxq_getchain(&pi->pi_key);
1787         UMTXQ_LOCKED_ASSERT(uc);
1788         TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink);
1789 }
1790
1791 /*
1792  * Lock a PI mutex.
1793  */
1794 static int
1795 do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags,
1796     struct _umtx_time *timeout, int try)
1797 {
1798         struct abs_timeout timo;
1799         struct umtx_q *uq;
1800         struct umtx_pi *pi, *new_pi;
1801         uint32_t id, old_owner, owner, old;
1802         int error, rv;
1803
1804         id = td->td_tid;
1805         uq = td->td_umtxq;
1806
1807         if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
1808             TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags),
1809             &uq->uq_key)) != 0)
1810                 return (error);
1811
1812         if (timeout != NULL)
1813                 abs_timeout_init2(&timo, timeout);
1814
1815         umtxq_lock(&uq->uq_key);
1816         pi = umtx_pi_lookup(&uq->uq_key);
1817         if (pi == NULL) {
1818                 new_pi = umtx_pi_alloc(M_NOWAIT);
1819                 if (new_pi == NULL) {
1820                         umtxq_unlock(&uq->uq_key);
1821                         new_pi = umtx_pi_alloc(M_WAITOK);
1822                         umtxq_lock(&uq->uq_key);
1823                         pi = umtx_pi_lookup(&uq->uq_key);
1824                         if (pi != NULL) {
1825                                 umtx_pi_free(new_pi);
1826                                 new_pi = NULL;
1827                         }
1828                 }
1829                 if (new_pi != NULL) {
1830                         new_pi->pi_key = uq->uq_key;
1831                         umtx_pi_insert(new_pi);
1832                         pi = new_pi;
1833                 }
1834         }
1835         umtx_pi_ref(pi);
1836         umtxq_unlock(&uq->uq_key);
1837
1838         /*
1839          * Care must be exercised when dealing with umtx structure.  It
1840          * can fault on any access.
1841          */
1842         for (;;) {
1843                 /*
1844                  * Try the uncontested case.  This should be done in userland.
1845                  */
1846                 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id);
1847                 /* The address was invalid. */
1848                 if (rv == -1) {
1849                         error = EFAULT;
1850                         break;
1851                 }
1852                 /* The acquire succeeded. */
1853                 if (rv == 0) {
1854                         MPASS(owner == UMUTEX_UNOWNED);
1855                         error = 0;
1856                         break;
1857                 }
1858
1859                 if (owner == UMUTEX_RB_NOTRECOV) {
1860                         error = ENOTRECOVERABLE;
1861                         break;
1862                 }
1863
1864                 /*
1865                  * Avoid overwriting a possible error from sleep due
1866                  * to the pending signal with suspension check result.
1867                  */
1868                 if (error == 0) {
1869                         error = thread_check_susp(td, true);
1870                         if (error != 0)
1871                                 break;
1872                 }
1873
1874                 /* If no one owns it but it is contested try to acquire it. */
1875                 if (owner == UMUTEX_CONTESTED || owner == UMUTEX_RB_OWNERDEAD) {
1876                         old_owner = owner;
1877                         rv = casueword32(&m->m_owner, owner, &owner,
1878                             id | UMUTEX_CONTESTED);
1879                         /* The address was invalid. */
1880                         if (rv == -1) {
1881                                 error = EFAULT;
1882                                 break;
1883                         }
1884                         if (rv == 1) {
1885                                 if (error == 0) {
1886                                         error = thread_check_susp(td, true);
1887                                         if (error != 0)
1888                                                 break;
1889                                 }
1890
1891                                 /*
1892                                  * If this failed the lock could
1893                                  * changed, restart.
1894                                  */
1895                                 continue;
1896                         }
1897
1898                         MPASS(rv == 0);
1899                         MPASS(owner == old_owner);
1900                         umtxq_lock(&uq->uq_key);
1901                         umtxq_busy(&uq->uq_key);
1902                         error = umtx_pi_claim(pi, td);
1903                         umtxq_unbusy(&uq->uq_key);
1904                         umtxq_unlock(&uq->uq_key);
1905                         if (error != 0) {
1906                                 /*
1907                                  * Since we're going to return an
1908                                  * error, restore the m_owner to its
1909                                  * previous, unowned state to avoid
1910                                  * compounding the problem.
1911                                  */
1912                                 (void)casuword32(&m->m_owner,
1913                                     id | UMUTEX_CONTESTED, old_owner);
1914                         }
1915                         if (error == 0 && old_owner == UMUTEX_RB_OWNERDEAD)
1916                                 error = EOWNERDEAD;
1917                         break;
1918                 }
1919
1920                 if ((owner & ~UMUTEX_CONTESTED) == id) {
1921                         error = EDEADLK;
1922                         break;
1923                 }
1924
1925                 if (try != 0) {
1926                         error = EBUSY;
1927                         break;
1928                 }
1929
1930                 /*
1931                  * If we caught a signal, we have retried and now
1932                  * exit immediately.
1933                  */
1934                 if (error != 0)
1935                         break;
1936
1937                 umtxq_lock(&uq->uq_key);
1938                 umtxq_busy(&uq->uq_key);
1939                 umtxq_unlock(&uq->uq_key);
1940
1941                 /*
1942                  * Set the contested bit so that a release in user space
1943                  * knows to use the system call for unlock.  If this fails
1944                  * either some one else has acquired the lock or it has been
1945                  * released.
1946                  */
1947                 rv = casueword32(&m->m_owner, owner, &old, owner |
1948                     UMUTEX_CONTESTED);
1949
1950                 /* The address was invalid. */
1951                 if (rv == -1) {
1952                         umtxq_unbusy_unlocked(&uq->uq_key);
1953                         error = EFAULT;
1954                         break;
1955                 }
1956                 if (rv == 1) {
1957                         umtxq_unbusy_unlocked(&uq->uq_key);
1958                         error = thread_check_susp(td, true);
1959                         if (error != 0)
1960                                 break;
1961
1962                         /*
1963                          * The lock changed and we need to retry or we
1964                          * lost a race to the thread unlocking the
1965                          * umtx.  Note that the UMUTEX_RB_OWNERDEAD
1966                          * value for owner is impossible there.
1967                          */
1968                         continue;
1969                 }
1970
1971                 umtxq_lock(&uq->uq_key);
1972
1973                 /* We set the contested bit, sleep. */
1974                 MPASS(old == owner);
1975                 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
1976                     "umtxpi", timeout == NULL ? NULL : &timo,
1977                     (flags & USYNC_PROCESS_SHARED) != 0);
1978                 if (error != 0)
1979                         continue;
1980
1981                 error = thread_check_susp(td, false);
1982                 if (error != 0)
1983                         break;
1984         }
1985
1986         umtxq_lock(&uq->uq_key);
1987         umtx_pi_unref(pi);
1988         umtxq_unlock(&uq->uq_key);
1989
1990         umtx_key_release(&uq->uq_key);
1991         return (error);
1992 }
1993
1994 /*
1995  * Unlock a PI mutex.
1996  */
1997 static int
1998 do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
1999 {
2000         struct umtx_key key;
2001         struct umtx_q *uq_first, *uq_first2, *uq_me;
2002         struct umtx_pi *pi, *pi2;
2003         uint32_t id, new_owner, old, owner;
2004         int count, error, pri;
2005
2006         id = td->td_tid;
2007
2008 usrloop:
2009         /*
2010          * Make sure we own this mtx.
2011          */
2012         error = fueword32(&m->m_owner, &owner);
2013         if (error == -1)
2014                 return (EFAULT);
2015
2016         if ((owner & ~UMUTEX_CONTESTED) != id)
2017                 return (EPERM);
2018
2019         new_owner = umtx_unlock_val(flags, rb);
2020
2021         /* This should be done in userland */
2022         if ((owner & UMUTEX_CONTESTED) == 0) {
2023                 error = casueword32(&m->m_owner, owner, &old, new_owner);
2024                 if (error == -1)
2025                         return (EFAULT);
2026                 if (error == 1) {
2027                         error = thread_check_susp(td, true);
2028                         if (error != 0)
2029                                 return (error);
2030                         goto usrloop;
2031                 }
2032                 if (old == owner)
2033                         return (0);
2034                 owner = old;
2035         }
2036
2037         /* We should only ever be in here for contested locks */
2038         if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2039             TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags),
2040             &key)) != 0)
2041                 return (error);
2042
2043         umtxq_lock(&key);
2044         umtxq_busy(&key);
2045         count = umtxq_count_pi(&key, &uq_first);
2046         if (uq_first != NULL) {
2047                 mtx_lock(&umtx_lock);
2048                 pi = uq_first->uq_pi_blocked;
2049                 KASSERT(pi != NULL, ("pi == NULL?"));
2050                 if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) {
2051                         mtx_unlock(&umtx_lock);
2052                         umtxq_unbusy(&key);
2053                         umtxq_unlock(&key);
2054                         umtx_key_release(&key);
2055                         /* userland messed the mutex */
2056                         return (EPERM);
2057                 }
2058                 uq_me = td->td_umtxq;
2059                 if (pi->pi_owner == td)
2060                         umtx_pi_disown(pi);
2061                 /* get highest priority thread which is still sleeping. */
2062                 uq_first = TAILQ_FIRST(&pi->pi_blocked);
2063                 while (uq_first != NULL &&
2064                     (uq_first->uq_flags & UQF_UMTXQ) == 0) {
2065                         uq_first = TAILQ_NEXT(uq_first, uq_lockq);
2066                 }
2067                 pri = PRI_MAX;
2068                 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
2069                         uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
2070                         if (uq_first2 != NULL) {
2071                                 if (pri > UPRI(uq_first2->uq_thread))
2072                                         pri = UPRI(uq_first2->uq_thread);
2073                         }
2074                 }
2075                 thread_lock(td);
2076                 sched_lend_user_prio(td, pri);
2077                 thread_unlock(td);
2078                 mtx_unlock(&umtx_lock);
2079                 if (uq_first)
2080                         umtxq_signal_thread(uq_first);
2081         } else {
2082                 pi = umtx_pi_lookup(&key);
2083                 /*
2084                  * A umtx_pi can exist if a signal or timeout removed the
2085                  * last waiter from the umtxq, but there is still
2086                  * a thread in do_lock_pi() holding the umtx_pi.
2087                  */
2088                 if (pi != NULL) {
2089                         /*
2090                          * The umtx_pi can be unowned, such as when a thread
2091                          * has just entered do_lock_pi(), allocated the
2092                          * umtx_pi, and unlocked the umtxq.
2093                          * If the current thread owns it, it must disown it.
2094                          */
2095                         mtx_lock(&umtx_lock);
2096                         if (pi->pi_owner == td)
2097                                 umtx_pi_disown(pi);
2098                         mtx_unlock(&umtx_lock);
2099                 }
2100         }
2101         umtxq_unlock(&key);
2102
2103         /*
2104          * When unlocking the umtx, it must be marked as unowned if
2105          * there is zero or one thread only waiting for it.
2106          * Otherwise, it must be marked as contested.
2107          */
2108
2109         if (count > 1)
2110                 new_owner |= UMUTEX_CONTESTED;
2111 again:
2112         error = casueword32(&m->m_owner, owner, &old, new_owner);
2113         if (error == 1) {
2114                 error = thread_check_susp(td, false);
2115                 if (error == 0)
2116                         goto again;
2117         }
2118         umtxq_unbusy_unlocked(&key);
2119         umtx_key_release(&key);
2120         if (error == -1)
2121                 return (EFAULT);
2122         if (error == 0 && old != owner)
2123                 return (EINVAL);
2124         return (error);
2125 }
2126
2127 /*
2128  * Lock a PP mutex.
2129  */
2130 static int
2131 do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags,
2132     struct _umtx_time *timeout, int try)
2133 {
2134         struct abs_timeout timo;
2135         struct umtx_q *uq, *uq2;
2136         struct umtx_pi *pi;
2137         uint32_t ceiling;
2138         uint32_t owner, id;
2139         int error, pri, old_inherited_pri, su, rv;
2140
2141         id = td->td_tid;
2142         uq = td->td_umtxq;
2143         if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2144             TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
2145             &uq->uq_key)) != 0)
2146                 return (error);
2147
2148         if (timeout != NULL)
2149                 abs_timeout_init2(&timo, timeout);
2150
2151         su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
2152         for (;;) {
2153                 old_inherited_pri = uq->uq_inherited_pri;
2154                 umtxq_lock(&uq->uq_key);
2155                 umtxq_busy(&uq->uq_key);
2156                 umtxq_unlock(&uq->uq_key);
2157
2158                 rv = fueword32(&m->m_ceilings[0], &ceiling);
2159                 if (rv == -1) {
2160                         error = EFAULT;
2161                         goto out;
2162                 }
2163                 ceiling = RTP_PRIO_MAX - ceiling;
2164                 if (ceiling > RTP_PRIO_MAX) {
2165                         error = EINVAL;
2166                         goto out;
2167                 }
2168
2169                 mtx_lock(&umtx_lock);
2170                 if (UPRI(td) < PRI_MIN_REALTIME + ceiling) {
2171                         mtx_unlock(&umtx_lock);
2172                         error = EINVAL;
2173                         goto out;
2174                 }
2175                 if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) {
2176                         uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling;
2177                         thread_lock(td);
2178                         if (uq->uq_inherited_pri < UPRI(td))
2179                                 sched_lend_user_prio(td, uq->uq_inherited_pri);
2180                         thread_unlock(td);
2181                 }
2182                 mtx_unlock(&umtx_lock);
2183
2184                 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
2185                     id | UMUTEX_CONTESTED);
2186                 /* The address was invalid. */
2187                 if (rv == -1) {
2188                         error = EFAULT;
2189                         break;
2190                 }
2191                 if (rv == 0) {
2192                         MPASS(owner == UMUTEX_CONTESTED);
2193                         error = 0;
2194                         break;
2195                 }
2196                 /* rv == 1 */
2197                 if (owner == UMUTEX_RB_OWNERDEAD) {
2198                         rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD,
2199                             &owner, id | UMUTEX_CONTESTED);
2200                         if (rv == -1) {
2201                                 error = EFAULT;
2202                                 break;
2203                         }
2204                         if (rv == 0) {
2205                                 MPASS(owner == UMUTEX_RB_OWNERDEAD);
2206                                 error = EOWNERDEAD; /* success */
2207                                 break;
2208                         }
2209
2210                         /*
2211                          *  rv == 1, only check for suspension if we
2212                          *  did not already catched a signal.  If we
2213                          *  get an error from the check, the same
2214                          *  condition is checked by the umtxq_sleep()
2215                          *  call below, so we should obliterate the
2216                          *  error to not skip the last loop iteration.
2217                          */
2218                         if (error == 0) {
2219                                 error = thread_check_susp(td, false);
2220                                 if (error == 0) {
2221                                         if (try != 0)
2222                                                 error = EBUSY;
2223                                         else
2224                                                 continue;
2225                                 }
2226                                 error = 0;
2227                         }
2228                 } else if (owner == UMUTEX_RB_NOTRECOV) {
2229                         error = ENOTRECOVERABLE;
2230                 }
2231
2232                 if (try != 0)
2233                         error = EBUSY;
2234
2235                 /*
2236                  * If we caught a signal, we have retried and now
2237                  * exit immediately.
2238                  */
2239                 if (error != 0)
2240                         break;
2241
2242                 umtxq_lock(&uq->uq_key);
2243                 umtxq_insert(uq);
2244                 umtxq_unbusy(&uq->uq_key);
2245                 error = umtxq_sleep(uq, "umtxpp", timeout == NULL ?
2246                     NULL : &timo);
2247                 umtxq_remove(uq);
2248                 umtxq_unlock(&uq->uq_key);
2249
2250                 mtx_lock(&umtx_lock);
2251                 uq->uq_inherited_pri = old_inherited_pri;
2252                 pri = PRI_MAX;
2253                 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2254                         uq2 = TAILQ_FIRST(&pi->pi_blocked);
2255                         if (uq2 != NULL) {
2256                                 if (pri > UPRI(uq2->uq_thread))
2257                                         pri = UPRI(uq2->uq_thread);
2258                         }
2259                 }
2260                 if (pri > uq->uq_inherited_pri)
2261                         pri = uq->uq_inherited_pri;
2262                 thread_lock(td);
2263                 sched_lend_user_prio(td, pri);
2264                 thread_unlock(td);
2265                 mtx_unlock(&umtx_lock);
2266         }
2267
2268         if (error != 0 && error != EOWNERDEAD) {
2269                 mtx_lock(&umtx_lock);
2270                 uq->uq_inherited_pri = old_inherited_pri;
2271                 pri = PRI_MAX;
2272                 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2273                         uq2 = TAILQ_FIRST(&pi->pi_blocked);
2274                         if (uq2 != NULL) {
2275                                 if (pri > UPRI(uq2->uq_thread))
2276                                         pri = UPRI(uq2->uq_thread);
2277                         }
2278                 }
2279                 if (pri > uq->uq_inherited_pri)
2280                         pri = uq->uq_inherited_pri;
2281                 thread_lock(td);
2282                 sched_lend_user_prio(td, pri);
2283                 thread_unlock(td);
2284                 mtx_unlock(&umtx_lock);
2285         }
2286
2287 out:
2288         umtxq_unbusy_unlocked(&uq->uq_key);
2289         umtx_key_release(&uq->uq_key);
2290         return (error);
2291 }
2292
2293 /*
2294  * Unlock a PP mutex.
2295  */
2296 static int
2297 do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, bool rb)
2298 {
2299         struct umtx_key key;
2300         struct umtx_q *uq, *uq2;
2301         struct umtx_pi *pi;
2302         uint32_t id, owner, rceiling;
2303         int error, pri, new_inherited_pri, su;
2304
2305         id = td->td_tid;
2306         uq = td->td_umtxq;
2307         su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0);
2308
2309         /*
2310          * Make sure we own this mtx.
2311          */
2312         error = fueword32(&m->m_owner, &owner);
2313         if (error == -1)
2314                 return (EFAULT);
2315
2316         if ((owner & ~UMUTEX_CONTESTED) != id)
2317                 return (EPERM);
2318
2319         error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t));
2320         if (error != 0)
2321                 return (error);
2322
2323         if (rceiling == -1)
2324                 new_inherited_pri = PRI_MAX;
2325         else {
2326                 rceiling = RTP_PRIO_MAX - rceiling;
2327                 if (rceiling > RTP_PRIO_MAX)
2328                         return (EINVAL);
2329                 new_inherited_pri = PRI_MIN_REALTIME + rceiling;
2330         }
2331
2332         if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2333             TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
2334             &key)) != 0)
2335                 return (error);
2336         umtxq_lock(&key);
2337         umtxq_busy(&key);
2338         umtxq_unlock(&key);
2339         /*
2340          * For priority protected mutex, always set unlocked state
2341          * to UMUTEX_CONTESTED, so that userland always enters kernel
2342          * to lock the mutex, it is necessary because thread priority
2343          * has to be adjusted for such mutex.
2344          */
2345         error = suword32(&m->m_owner, umtx_unlock_val(flags, rb) |
2346             UMUTEX_CONTESTED);
2347
2348         umtxq_lock(&key);
2349         if (error == 0)
2350                 umtxq_signal(&key, 1);
2351         umtxq_unbusy(&key);
2352         umtxq_unlock(&key);
2353
2354         if (error == -1)
2355                 error = EFAULT;
2356         else {
2357                 mtx_lock(&umtx_lock);
2358                 if (su != 0)
2359                         uq->uq_inherited_pri = new_inherited_pri;
2360                 pri = PRI_MAX;
2361                 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2362                         uq2 = TAILQ_FIRST(&pi->pi_blocked);
2363                         if (uq2 != NULL) {
2364                                 if (pri > UPRI(uq2->uq_thread))
2365                                         pri = UPRI(uq2->uq_thread);
2366                         }
2367                 }
2368                 if (pri > uq->uq_inherited_pri)
2369                         pri = uq->uq_inherited_pri;
2370                 thread_lock(td);
2371                 sched_lend_user_prio(td, pri);
2372                 thread_unlock(td);
2373                 mtx_unlock(&umtx_lock);
2374         }
2375         umtx_key_release(&key);
2376         return (error);
2377 }
2378
2379 static int
2380 do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
2381     uint32_t *old_ceiling)
2382 {
2383         struct umtx_q *uq;
2384         uint32_t flags, id, owner, save_ceiling;
2385         int error, rv, rv1;
2386
2387         error = fueword32(&m->m_flags, &flags);
2388         if (error == -1)
2389                 return (EFAULT);
2390         if ((flags & UMUTEX_PRIO_PROTECT) == 0)
2391                 return (EINVAL);
2392         if (ceiling > RTP_PRIO_MAX)
2393                 return (EINVAL);
2394         id = td->td_tid;
2395         uq = td->td_umtxq;
2396         if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ?
2397             TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags),
2398             &uq->uq_key)) != 0)
2399                 return (error);
2400         for (;;) {
2401                 umtxq_lock(&uq->uq_key);
2402                 umtxq_busy(&uq->uq_key);
2403                 umtxq_unlock(&uq->uq_key);
2404
2405                 rv = fueword32(&m->m_ceilings[0], &save_ceiling);
2406                 if (rv == -1) {
2407                         error = EFAULT;
2408                         break;
2409                 }
2410
2411                 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
2412                     id | UMUTEX_CONTESTED);
2413                 if (rv == -1) {
2414                         error = EFAULT;
2415                         break;
2416                 }
2417
2418                 if (rv == 0) {
2419                         MPASS(owner == UMUTEX_CONTESTED);
2420                         rv = suword32(&m->m_ceilings[0], ceiling);
2421                         rv1 = suword32(&m->m_owner, UMUTEX_CONTESTED);
2422                         error = (rv == 0 && rv1 == 0) ? 0: EFAULT;
2423                         break;
2424                 }
2425
2426                 if ((owner & ~UMUTEX_CONTESTED) == id) {
2427                         rv = suword32(&m->m_ceilings[0], ceiling);
2428                         error = rv == 0 ? 0 : EFAULT;
2429                         break;
2430                 }
2431
2432                 if (owner == UMUTEX_RB_OWNERDEAD) {
2433                         error = EOWNERDEAD;
2434                         break;
2435                 } else if (owner == UMUTEX_RB_NOTRECOV) {
2436                         error = ENOTRECOVERABLE;
2437                         break;
2438                 }
2439
2440                 /*
2441                  * If we caught a signal, we have retried and now
2442                  * exit immediately.
2443                  */
2444                 if (error != 0)
2445                         break;
2446
2447                 /*
2448                  * We set the contested bit, sleep. Otherwise the lock changed
2449                  * and we need to retry or we lost a race to the thread
2450                  * unlocking the umtx.
2451                  */
2452                 umtxq_lock(&uq->uq_key);
2453                 umtxq_insert(uq);
2454                 umtxq_unbusy(&uq->uq_key);
2455                 error = umtxq_sleep(uq, "umtxpp", NULL);
2456                 umtxq_remove(uq);
2457                 umtxq_unlock(&uq->uq_key);
2458         }
2459         umtxq_lock(&uq->uq_key);
2460         if (error == 0)
2461                 umtxq_signal(&uq->uq_key, INT_MAX);
2462         umtxq_unbusy(&uq->uq_key);
2463         umtxq_unlock(&uq->uq_key);
2464         umtx_key_release(&uq->uq_key);
2465         if (error == 0 && old_ceiling != NULL) {
2466                 rv = suword32(old_ceiling, save_ceiling);
2467                 error = rv == 0 ? 0 : EFAULT;
2468         }
2469         return (error);
2470 }
2471
2472 /*
2473  * Lock a userland POSIX mutex.
2474  */
2475 static int
2476 do_lock_umutex(struct thread *td, struct umutex *m,
2477     struct _umtx_time *timeout, int mode)
2478 {
2479         uint32_t flags;
2480         int error;
2481
2482         error = fueword32(&m->m_flags, &flags);
2483         if (error == -1)
2484                 return (EFAULT);
2485
2486         switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2487         case 0:
2488                 error = do_lock_normal(td, m, flags, timeout, mode);
2489                 break;
2490         case UMUTEX_PRIO_INHERIT:
2491                 error = do_lock_pi(td, m, flags, timeout, mode);
2492                 break;
2493         case UMUTEX_PRIO_PROTECT:
2494                 error = do_lock_pp(td, m, flags, timeout, mode);
2495                 break;
2496         default:
2497                 return (EINVAL);
2498         }
2499         if (timeout == NULL) {
2500                 if (error == EINTR && mode != _UMUTEX_WAIT)
2501                         error = ERESTART;
2502         } else {
2503                 /* Timed-locking is not restarted. */
2504                 if (error == ERESTART)
2505                         error = EINTR;
2506         }
2507         return (error);
2508 }
2509
2510 /*
2511  * Unlock a userland POSIX mutex.
2512  */
2513 static int
2514 do_unlock_umutex(struct thread *td, struct umutex *m, bool rb)
2515 {
2516         uint32_t flags;
2517         int error;
2518
2519         error = fueword32(&m->m_flags, &flags);
2520         if (error == -1)
2521                 return (EFAULT);
2522
2523         switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
2524         case 0:
2525                 return (do_unlock_normal(td, m, flags, rb));
2526         case UMUTEX_PRIO_INHERIT:
2527                 return (do_unlock_pi(td, m, flags, rb));
2528         case UMUTEX_PRIO_PROTECT:
2529                 return (do_unlock_pp(td, m, flags, rb));
2530         }
2531
2532         return (EINVAL);
2533 }
2534
2535 static int
2536 do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
2537     struct timespec *timeout, u_long wflags)
2538 {
2539         struct abs_timeout timo;
2540         struct umtx_q *uq;
2541         uint32_t flags, clockid, hasw;
2542         int error;
2543
2544         uq = td->td_umtxq;
2545         error = fueword32(&cv->c_flags, &flags);
2546         if (error == -1)
2547                 return (EFAULT);
2548         error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2549         if (error != 0)
2550                 return (error);
2551
2552         if ((wflags & CVWAIT_CLOCKID) != 0) {
2553                 error = fueword32(&cv->c_clockid, &clockid);
2554                 if (error == -1) {
2555                         umtx_key_release(&uq->uq_key);
2556                         return (EFAULT);
2557                 }
2558                 if (clockid < CLOCK_REALTIME ||
2559                     clockid >= CLOCK_THREAD_CPUTIME_ID) {
2560                         /* hmm, only HW clock id will work. */
2561                         umtx_key_release(&uq->uq_key);
2562                         return (EINVAL);
2563                 }
2564         } else {
2565                 clockid = CLOCK_REALTIME;
2566         }
2567
2568         umtxq_lock(&uq->uq_key);
2569         umtxq_busy(&uq->uq_key);
2570         umtxq_insert(uq);
2571         umtxq_unlock(&uq->uq_key);
2572
2573         /*
2574          * Set c_has_waiters to 1 before releasing user mutex, also
2575          * don't modify cache line when unnecessary.
2576          */
2577         error = fueword32(&cv->c_has_waiters, &hasw);
2578         if (error == 0 && hasw == 0)
2579                 suword32(&cv->c_has_waiters, 1);
2580
2581         umtxq_unbusy_unlocked(&uq->uq_key);
2582
2583         error = do_unlock_umutex(td, m, false);
2584
2585         if (timeout != NULL)
2586                 abs_timeout_init(&timo, clockid, (wflags & CVWAIT_ABSTIME) != 0,
2587                     timeout);
2588
2589         umtxq_lock(&uq->uq_key);
2590         if (error == 0) {
2591                 error = umtxq_sleep(uq, "ucond", timeout == NULL ?
2592                     NULL : &timo);
2593         }
2594
2595         if ((uq->uq_flags & UQF_UMTXQ) == 0)
2596                 error = 0;
2597         else {
2598                 /*
2599                  * This must be timeout,interrupted by signal or
2600                  * surprious wakeup, clear c_has_waiter flag when
2601                  * necessary.
2602                  */
2603                 umtxq_busy(&uq->uq_key);
2604                 if ((uq->uq_flags & UQF_UMTXQ) != 0) {
2605                         int oldlen = uq->uq_cur_queue->length;
2606                         umtxq_remove(uq);
2607                         if (oldlen == 1) {
2608                                 umtxq_unlock(&uq->uq_key);
2609                                 suword32(&cv->c_has_waiters, 0);
2610                                 umtxq_lock(&uq->uq_key);
2611                         }
2612                 }
2613                 umtxq_unbusy(&uq->uq_key);
2614                 if (error == ERESTART)
2615                         error = EINTR;
2616         }
2617
2618         umtxq_unlock(&uq->uq_key);
2619         umtx_key_release(&uq->uq_key);
2620         return (error);
2621 }
2622
2623 /*
2624  * Signal a userland condition variable.
2625  */
2626 static int
2627 do_cv_signal(struct thread *td, struct ucond *cv)
2628 {
2629         struct umtx_key key;
2630         int error, cnt, nwake;
2631         uint32_t flags;
2632
2633         error = fueword32(&cv->c_flags, &flags);
2634         if (error == -1)
2635                 return (EFAULT);
2636         if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
2637                 return (error);
2638         umtxq_lock(&key);
2639         umtxq_busy(&key);
2640         cnt = umtxq_count(&key);
2641         nwake = umtxq_signal(&key, 1);
2642         if (cnt <= nwake) {
2643                 umtxq_unlock(&key);
2644                 error = suword32(&cv->c_has_waiters, 0);
2645                 if (error == -1)
2646                         error = EFAULT;
2647                 umtxq_lock(&key);
2648         }
2649         umtxq_unbusy(&key);
2650         umtxq_unlock(&key);
2651         umtx_key_release(&key);
2652         return (error);
2653 }
2654
2655 static int
2656 do_cv_broadcast(struct thread *td, struct ucond *cv)
2657 {
2658         struct umtx_key key;
2659         int error;
2660         uint32_t flags;
2661
2662         error = fueword32(&cv->c_flags, &flags);
2663         if (error == -1)
2664                 return (EFAULT);
2665         if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
2666                 return (error);
2667
2668         umtxq_lock(&key);
2669         umtxq_busy(&key);
2670         umtxq_signal(&key, INT_MAX);
2671         umtxq_unlock(&key);
2672
2673         error = suword32(&cv->c_has_waiters, 0);
2674         if (error == -1)
2675                 error = EFAULT;
2676
2677         umtxq_unbusy_unlocked(&key);
2678
2679         umtx_key_release(&key);
2680         return (error);
2681 }
2682
2683 static int
2684 do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag,
2685     struct _umtx_time *timeout)
2686 {
2687         struct abs_timeout timo;
2688         struct umtx_q *uq;
2689         uint32_t flags, wrflags;
2690         int32_t state, oldstate;
2691         int32_t blocked_readers;
2692         int error, error1, rv;
2693
2694         uq = td->td_umtxq;
2695         error = fueword32(&rwlock->rw_flags, &flags);
2696         if (error == -1)
2697                 return (EFAULT);
2698         error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2699         if (error != 0)
2700                 return (error);
2701
2702         if (timeout != NULL)
2703                 abs_timeout_init2(&timo, timeout);
2704
2705         wrflags = URWLOCK_WRITE_OWNER;
2706         if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER))
2707                 wrflags |= URWLOCK_WRITE_WAITERS;
2708
2709         for (;;) {
2710                 rv = fueword32(&rwlock->rw_state, &state);
2711                 if (rv == -1) {
2712                         umtx_key_release(&uq->uq_key);
2713                         return (EFAULT);
2714                 }
2715
2716                 /* try to lock it */
2717                 while (!(state & wrflags)) {
2718                         if (__predict_false(URWLOCK_READER_COUNT(state) ==
2719                             URWLOCK_MAX_READERS)) {
2720                                 umtx_key_release(&uq->uq_key);
2721                                 return (EAGAIN);
2722                         }
2723                         rv = casueword32(&rwlock->rw_state, state,
2724                             &oldstate, state + 1);
2725                         if (rv == -1) {
2726                                 umtx_key_release(&uq->uq_key);
2727                                 return (EFAULT);
2728                         }
2729                         if (rv == 0) {
2730                                 MPASS(oldstate == state);
2731                                 umtx_key_release(&uq->uq_key);
2732                                 return (0);
2733                         }
2734                         error = thread_check_susp(td, true);
2735                         if (error != 0)
2736                                 break;
2737                         state = oldstate;
2738                 }
2739
2740                 if (error)
2741                         break;
2742
2743                 /* grab monitor lock */
2744                 umtxq_lock(&uq->uq_key);
2745                 umtxq_busy(&uq->uq_key);
2746                 umtxq_unlock(&uq->uq_key);
2747
2748                 /*
2749                  * re-read the state, in case it changed between the try-lock above
2750                  * and the check below
2751                  */
2752                 rv = fueword32(&rwlock->rw_state, &state);
2753                 if (rv == -1)
2754                         error = EFAULT;
2755
2756                 /* set read contention bit */
2757                 while (error == 0 && (state & wrflags) &&
2758                     !(state & URWLOCK_READ_WAITERS)) {
2759                         rv = casueword32(&rwlock->rw_state, state,
2760                             &oldstate, state | URWLOCK_READ_WAITERS);
2761                         if (rv == -1) {
2762                                 error = EFAULT;
2763                                 break;
2764                         }
2765                         if (rv == 0) {
2766                                 MPASS(oldstate == state);
2767                                 goto sleep;
2768                         }
2769                         state = oldstate;
2770                         error = thread_check_susp(td, false);
2771                         if (error != 0)
2772                                 break;
2773                 }
2774                 if (error != 0) {
2775                         umtxq_unbusy_unlocked(&uq->uq_key);
2776                         break;
2777                 }
2778
2779                 /* state is changed while setting flags, restart */
2780                 if (!(state & wrflags)) {
2781                         umtxq_unbusy_unlocked(&uq->uq_key);
2782                         error = thread_check_susp(td, true);
2783                         if (error != 0)
2784                                 break;
2785                         continue;
2786                 }
2787
2788 sleep:
2789                 /*
2790                  * Contention bit is set, before sleeping, increase
2791                  * read waiter count.
2792                  */
2793                 rv = fueword32(&rwlock->rw_blocked_readers,
2794                     &blocked_readers);
2795                 if (rv == -1) {
2796                         umtxq_unbusy_unlocked(&uq->uq_key);
2797                         error = EFAULT;
2798                         break;
2799                 }
2800                 suword32(&rwlock->rw_blocked_readers, blocked_readers+1);
2801
2802                 while (state & wrflags) {
2803                         umtxq_lock(&uq->uq_key);
2804                         umtxq_insert(uq);
2805                         umtxq_unbusy(&uq->uq_key);
2806
2807                         error = umtxq_sleep(uq, "urdlck", timeout == NULL ?
2808                             NULL : &timo);
2809
2810                         umtxq_busy(&uq->uq_key);
2811                         umtxq_remove(uq);
2812                         umtxq_unlock(&uq->uq_key);
2813                         if (error)
2814                                 break;
2815                         rv = fueword32(&rwlock->rw_state, &state);
2816                         if (rv == -1) {
2817                                 error = EFAULT;
2818                                 break;
2819                         }
2820                 }
2821
2822                 /* decrease read waiter count, and may clear read contention bit */
2823                 rv = fueword32(&rwlock->rw_blocked_readers,
2824                     &blocked_readers);
2825                 if (rv == -1) {
2826                         umtxq_unbusy_unlocked(&uq->uq_key);
2827                         error = EFAULT;
2828                         break;
2829                 }
2830                 suword32(&rwlock->rw_blocked_readers, blocked_readers-1);
2831                 if (blocked_readers == 1) {
2832                         rv = fueword32(&rwlock->rw_state, &state);
2833                         if (rv == -1) {
2834                                 umtxq_unbusy_unlocked(&uq->uq_key);
2835                                 error = EFAULT;
2836                                 break;
2837                         }
2838                         for (;;) {
2839                                 rv = casueword32(&rwlock->rw_state, state,
2840                                     &oldstate, state & ~URWLOCK_READ_WAITERS);
2841                                 if (rv == -1) {
2842                                         error = EFAULT;
2843                                         break;
2844                                 }
2845                                 if (rv == 0) {
2846                                         MPASS(oldstate == state);
2847                                         break;
2848                                 }
2849                                 state = oldstate;
2850                                 error1 = thread_check_susp(td, false);
2851                                 if (error1 != 0) {
2852                                         if (error == 0)
2853                                                 error = error1;
2854                                         break;
2855                                 }
2856                         }
2857                 }
2858
2859                 umtxq_unbusy_unlocked(&uq->uq_key);
2860                 if (error != 0)
2861                         break;
2862         }
2863         umtx_key_release(&uq->uq_key);
2864         if (error == ERESTART)
2865                 error = EINTR;
2866         return (error);
2867 }
2868
2869 static int
2870 do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeout)
2871 {
2872         struct abs_timeout timo;
2873         struct umtx_q *uq;
2874         uint32_t flags;
2875         int32_t state, oldstate;
2876         int32_t blocked_writers;
2877         int32_t blocked_readers;
2878         int error, error1, rv;
2879
2880         uq = td->td_umtxq;
2881         error = fueword32(&rwlock->rw_flags, &flags);
2882         if (error == -1)
2883                 return (EFAULT);
2884         error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
2885         if (error != 0)
2886                 return (error);
2887
2888         if (timeout != NULL)
2889                 abs_timeout_init2(&timo, timeout);
2890
2891         blocked_readers = 0;
2892         for (;;) {
2893                 rv = fueword32(&rwlock->rw_state, &state);
2894                 if (rv == -1) {
2895                         umtx_key_release(&uq->uq_key);
2896                         return (EFAULT);
2897                 }
2898                 while ((state & URWLOCK_WRITE_OWNER) == 0 &&
2899                     URWLOCK_READER_COUNT(state) == 0) {
2900                         rv = casueword32(&rwlock->rw_state, state,
2901                             &oldstate, state | URWLOCK_WRITE_OWNER);
2902                         if (rv == -1) {
2903                                 umtx_key_release(&uq->uq_key);
2904                                 return (EFAULT);
2905                         }
2906                         if (rv == 0) {
2907                                 MPASS(oldstate == state);
2908                                 umtx_key_release(&uq->uq_key);
2909                                 return (0);
2910                         }
2911                         state = oldstate;
2912                         error = thread_check_susp(td, true);
2913                         if (error != 0)
2914                                 break;
2915                 }
2916
2917                 if (error) {
2918                         if ((state & (URWLOCK_WRITE_OWNER |
2919                             URWLOCK_WRITE_WAITERS)) == 0 &&
2920                             blocked_readers != 0) {
2921                                 umtxq_lock(&uq->uq_key);
2922                                 umtxq_busy(&uq->uq_key);
2923                                 umtxq_signal_queue(&uq->uq_key, INT_MAX,
2924                                     UMTX_SHARED_QUEUE);
2925                                 umtxq_unbusy(&uq->uq_key);
2926                                 umtxq_unlock(&uq->uq_key);
2927                         }
2928
2929                         break;
2930                 }
2931
2932                 /* grab monitor lock */
2933                 umtxq_lock(&uq->uq_key);
2934                 umtxq_busy(&uq->uq_key);
2935                 umtxq_unlock(&uq->uq_key);
2936
2937                 /*
2938                  * Re-read the state, in case it changed between the
2939                  * try-lock above and the check below.
2940                  */
2941                 rv = fueword32(&rwlock->rw_state, &state);
2942                 if (rv == -1)
2943                         error = EFAULT;
2944
2945                 while (error == 0 && ((state & URWLOCK_WRITE_OWNER) ||
2946                     URWLOCK_READER_COUNT(state) != 0) &&
2947                     (state & URWLOCK_WRITE_WAITERS) == 0) {
2948                         rv = casueword32(&rwlock->rw_state, state,
2949                             &oldstate, state | URWLOCK_WRITE_WAITERS);
2950                         if (rv == -1) {
2951                                 error = EFAULT;
2952                                 break;
2953                         }
2954                         if (rv == 0) {
2955                                 MPASS(oldstate == state);
2956                                 goto sleep;
2957                         }
2958                         state = oldstate;
2959                         error = thread_check_susp(td, false);
2960                         if (error != 0)
2961                                 break;
2962                 }
2963                 if (error != 0) {
2964                         umtxq_unbusy_unlocked(&uq->uq_key);
2965                         break;
2966                 }
2967
2968                 if ((state & URWLOCK_WRITE_OWNER) == 0 &&
2969                     URWLOCK_READER_COUNT(state) == 0) {
2970                         umtxq_unbusy_unlocked(&uq->uq_key);
2971                         error = thread_check_susp(td, false);
2972                         if (error != 0)
2973                                 break;
2974                         continue;
2975                 }
2976 sleep:
2977                 rv = fueword32(&rwlock->rw_blocked_writers,
2978                     &blocked_writers);
2979                 if (rv == -1) {
2980                         umtxq_unbusy_unlocked(&uq->uq_key);
2981                         error = EFAULT;
2982                         break;
2983                 }
2984                 suword32(&rwlock->rw_blocked_writers, blocked_writers + 1);
2985
2986                 while ((state & URWLOCK_WRITE_OWNER) ||
2987                     URWLOCK_READER_COUNT(state) != 0) {
2988                         umtxq_lock(&uq->uq_key);
2989                         umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE);
2990                         umtxq_unbusy(&uq->uq_key);
2991
2992                         error = umtxq_sleep(uq, "uwrlck", timeout == NULL ?
2993                             NULL : &timo);
2994
2995                         umtxq_busy(&uq->uq_key);
2996                         umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE);
2997                         umtxq_unlock(&uq->uq_key);
2998                         if (error)
2999                                 break;
3000                         rv = fueword32(&rwlock->rw_state, &state);
3001                         if (rv == -1) {
3002                                 error = EFAULT;
3003                                 break;
3004                         }
3005                 }
3006
3007                 rv = fueword32(&rwlock->rw_blocked_writers,
3008                     &blocked_writers);
3009                 if (rv == -1) {
3010                         umtxq_unbusy_unlocked(&uq->uq_key);
3011                         error = EFAULT;
3012                         break;
3013                 }
3014                 suword32(&rwlock->rw_blocked_writers, blocked_writers-1);
3015                 if (blocked_writers == 1) {
3016                         rv = fueword32(&rwlock->rw_state, &state);
3017                         if (rv == -1) {
3018                                 umtxq_unbusy_unlocked(&uq->uq_key);
3019                                 error = EFAULT;
3020                                 break;
3021                         }
3022                         for (;;) {
3023                                 rv = casueword32(&rwlock->rw_state, state,
3024                                     &oldstate, state & ~URWLOCK_WRITE_WAITERS);
3025                                 if (rv == -1) {
3026                                         error = EFAULT;
3027                                         break;
3028                                 }
3029                                 if (rv == 0) {
3030                                         MPASS(oldstate == state);
3031                                         break;
3032                                 }
3033                                 state = oldstate;
3034                                 error1 = thread_check_susp(td, false);
3035                                 /*
3036                                  * We are leaving the URWLOCK_WRITE_WAITERS
3037                                  * behind, but this should not harm the
3038                                  * correctness.
3039                                  */
3040                                 if (error1 != 0) {
3041                                         if (error == 0)
3042                                                 error = error1;
3043                                         break;
3044                                 }
3045                         }
3046                         rv = fueword32(&rwlock->rw_blocked_readers,
3047                             &blocked_readers);
3048                         if (rv == -1) {
3049                                 umtxq_unbusy_unlocked(&uq->uq_key);
3050                                 error = EFAULT;
3051                                 break;
3052                         }
3053                 } else
3054                         blocked_readers = 0;
3055
3056                 umtxq_unbusy_unlocked(&uq->uq_key);
3057         }
3058
3059         umtx_key_release(&uq->uq_key);
3060         if (error == ERESTART)
3061                 error = EINTR;
3062         return (error);
3063 }
3064
3065 static int
3066 do_rw_unlock(struct thread *td, struct urwlock *rwlock)
3067 {
3068         struct umtx_q *uq;
3069         uint32_t flags;
3070         int32_t state, oldstate;
3071         int error, rv, q, count;
3072
3073         uq = td->td_umtxq;
3074         error = fueword32(&rwlock->rw_flags, &flags);
3075         if (error == -1)
3076                 return (EFAULT);
3077         error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3078         if (error != 0)
3079                 return (error);
3080
3081         error = fueword32(&rwlock->rw_state, &state);
3082         if (error == -1) {
3083                 error = EFAULT;
3084                 goto out;
3085         }
3086         if (state & URWLOCK_WRITE_OWNER) {
3087                 for (;;) {
3088                         rv = casueword32(&rwlock->rw_state, state,
3089                             &oldstate, state & ~URWLOCK_WRITE_OWNER);
3090                         if (rv == -1) {
3091                                 error = EFAULT;
3092                                 goto out;
3093                         }
3094                         if (rv == 1) {
3095                                 state = oldstate;
3096                                 if (!(oldstate & URWLOCK_WRITE_OWNER)) {
3097                                         error = EPERM;
3098                                         goto out;
3099                                 }
3100                                 error = thread_check_susp(td, true);
3101                                 if (error != 0)
3102                                         goto out;
3103                         } else
3104                                 break;
3105                 }
3106         } else if (URWLOCK_READER_COUNT(state) != 0) {
3107                 for (;;) {
3108                         rv = casueword32(&rwlock->rw_state, state,
3109                             &oldstate, state - 1);
3110                         if (rv == -1) {
3111                                 error = EFAULT;
3112                                 goto out;
3113                         }
3114                         if (rv == 1) {
3115                                 state = oldstate;
3116                                 if (URWLOCK_READER_COUNT(oldstate) == 0) {
3117                                         error = EPERM;
3118                                         goto out;
3119                                 }
3120                                 error = thread_check_susp(td, true);
3121                                 if (error != 0)
3122                                         goto out;
3123                         } else
3124                                 break;
3125                 }
3126         } else {
3127                 error = EPERM;
3128                 goto out;
3129         }
3130
3131         count = 0;
3132
3133         if (!(flags & URWLOCK_PREFER_READER)) {
3134                 if (state & URWLOCK_WRITE_WAITERS) {
3135                         count = 1;
3136                         q = UMTX_EXCLUSIVE_QUEUE;
3137                 } else if (state & URWLOCK_READ_WAITERS) {
3138                         count = INT_MAX;
3139                         q = UMTX_SHARED_QUEUE;
3140                 }
3141         } else {
3142                 if (state & URWLOCK_READ_WAITERS) {
3143                         count = INT_MAX;
3144                         q = UMTX_SHARED_QUEUE;
3145                 } else if (state & URWLOCK_WRITE_WAITERS) {
3146                         count = 1;
3147                         q = UMTX_EXCLUSIVE_QUEUE;
3148                 }
3149         }
3150
3151         if (count) {
3152                 umtxq_lock(&uq->uq_key);
3153                 umtxq_busy(&uq->uq_key);
3154                 umtxq_signal_queue(&uq->uq_key, count, q);
3155                 umtxq_unbusy(&uq->uq_key);
3156                 umtxq_unlock(&uq->uq_key);
3157         }
3158 out:
3159         umtx_key_release(&uq->uq_key);
3160         return (error);
3161 }
3162
3163 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
3164 static int
3165 do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout)
3166 {
3167         struct abs_timeout timo;
3168         struct umtx_q *uq;
3169         uint32_t flags, count, count1;
3170         int error, rv, rv1;
3171
3172         uq = td->td_umtxq;
3173         error = fueword32(&sem->_flags, &flags);
3174         if (error == -1)
3175                 return (EFAULT);
3176         error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3177         if (error != 0)
3178                 return (error);
3179
3180         if (timeout != NULL)
3181                 abs_timeout_init2(&timo, timeout);
3182
3183 again:
3184         umtxq_lock(&uq->uq_key);
3185         umtxq_busy(&uq->uq_key);
3186         umtxq_insert(uq);
3187         umtxq_unlock(&uq->uq_key);
3188         rv = casueword32(&sem->_has_waiters, 0, &count1, 1);
3189         if (rv == 0)
3190                 rv1 = fueword32(&sem->_count, &count);
3191         if (rv == -1 || (rv == 0 && (rv1 == -1 || count != 0)) ||
3192             (rv == 1 && count1 == 0)) {
3193                 umtxq_lock(&uq->uq_key);
3194                 umtxq_unbusy(&uq->uq_key);
3195                 umtxq_remove(uq);
3196                 umtxq_unlock(&uq->uq_key);
3197                 if (rv == 1) {
3198                         rv = thread_check_susp(td, true);
3199                         if (rv == 0)
3200                                 goto again;
3201                         error = rv;
3202                         goto out;
3203                 }
3204                 if (rv == 0)
3205                         rv = rv1;
3206                 error = rv == -1 ? EFAULT : 0;
3207                 goto out;
3208         }
3209         umtxq_lock(&uq->uq_key);
3210         umtxq_unbusy(&uq->uq_key);
3211
3212         error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
3213
3214         if ((uq->uq_flags & UQF_UMTXQ) == 0)
3215                 error = 0;
3216         else {
3217                 umtxq_remove(uq);
3218                 /* A relative timeout cannot be restarted. */
3219                 if (error == ERESTART && timeout != NULL &&
3220                     (timeout->_flags & UMTX_ABSTIME) == 0)
3221                         error = EINTR;
3222         }
3223         umtxq_unlock(&uq->uq_key);
3224 out:
3225         umtx_key_release(&uq->uq_key);
3226         return (error);
3227 }
3228
3229 /*
3230  * Signal a userland semaphore.
3231  */
3232 static int
3233 do_sem_wake(struct thread *td, struct _usem *sem)
3234 {
3235         struct umtx_key key;
3236         int error, cnt;
3237         uint32_t flags;
3238
3239         error = fueword32(&sem->_flags, &flags);
3240         if (error == -1)
3241                 return (EFAULT);
3242         if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
3243                 return (error);
3244         umtxq_lock(&key);
3245         umtxq_busy(&key);
3246         cnt = umtxq_count(&key);
3247         if (cnt > 0) {
3248                 /*
3249                  * Check if count is greater than 0, this means the memory is
3250                  * still being referenced by user code, so we can safely
3251                  * update _has_waiters flag.
3252                  */
3253                 if (cnt == 1) {
3254                         umtxq_unlock(&key);
3255                         error = suword32(&sem->_has_waiters, 0);
3256                         umtxq_lock(&key);
3257                         if (error == -1)
3258                                 error = EFAULT;
3259                 }
3260                 umtxq_signal(&key, 1);
3261         }
3262         umtxq_unbusy(&key);
3263         umtxq_unlock(&key);
3264         umtx_key_release(&key);
3265         return (error);
3266 }
3267 #endif
3268
3269 static int
3270 do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout)
3271 {
3272         struct abs_timeout timo;
3273         struct umtx_q *uq;
3274         uint32_t count, flags;
3275         int error, rv;
3276
3277         uq = td->td_umtxq;
3278         flags = fuword32(&sem->_flags);
3279         if (timeout != NULL)
3280                 abs_timeout_init2(&timo, timeout);
3281
3282 again:
3283         error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3284         if (error != 0)
3285                 return (error);
3286         umtxq_lock(&uq->uq_key);
3287         umtxq_busy(&uq->uq_key);
3288         umtxq_insert(uq);
3289         umtxq_unlock(&uq->uq_key);
3290         rv = fueword32(&sem->_count, &count);
3291         if (rv == -1) {
3292                 umtxq_lock(&uq->uq_key);
3293                 umtxq_unbusy(&uq->uq_key);
3294                 umtxq_remove(uq);
3295                 umtxq_unlock(&uq->uq_key);
3296                 umtx_key_release(&uq->uq_key);
3297                 return (EFAULT);
3298         }
3299         for (;;) {
3300                 if (USEM_COUNT(count) != 0) {
3301                         umtxq_lock(&uq->uq_key);
3302                         umtxq_unbusy(&uq->uq_key);
3303                         umtxq_remove(uq);
3304                         umtxq_unlock(&uq->uq_key);
3305                         umtx_key_release(&uq->uq_key);
3306                         return (0);
3307                 }
3308                 if (count == USEM_HAS_WAITERS)
3309                         break;
3310                 rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS);
3311                 if (rv == 0)
3312                         break;
3313                 umtxq_lock(&uq->uq_key);
3314                 umtxq_unbusy(&uq->uq_key);
3315                 umtxq_remove(uq);
3316                 umtxq_unlock(&uq->uq_key);
3317                 umtx_key_release(&uq->uq_key);
3318                 if (rv == -1)
3319                         return (EFAULT);
3320                 rv = thread_check_susp(td, true);
3321                 if (rv != 0)
3322                         return (rv);
3323                 goto again;
3324         }
3325         umtxq_lock(&uq->uq_key);
3326         umtxq_unbusy(&uq->uq_key);
3327
3328         error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
3329
3330         if ((uq->uq_flags & UQF_UMTXQ) == 0)
3331                 error = 0;
3332         else {
3333                 umtxq_remove(uq);
3334                 if (timeout != NULL && (timeout->_flags & UMTX_ABSTIME) == 0) {
3335                         /* A relative timeout cannot be restarted. */
3336                         if (error == ERESTART)
3337                                 error = EINTR;
3338                         if (error == EINTR) {
3339                                 abs_timeout_update(&timo);
3340                                 timespecsub(&timo.end, &timo.cur,
3341                                     &timeout->_timeout);
3342                         }
3343                 }
3344         }
3345         umtxq_unlock(&uq->uq_key);
3346         umtx_key_release(&uq->uq_key);
3347         return (error);
3348 }
3349
3350 /*
3351  * Signal a userland semaphore.
3352  */
3353 static int
3354 do_sem2_wake(struct thread *td, struct _usem2 *sem)
3355 {
3356         struct umtx_key key;
3357         int error, cnt, rv;
3358         uint32_t count, flags;
3359
3360         rv = fueword32(&sem->_flags, &flags);
3361         if (rv == -1)
3362                 return (EFAULT);
3363         if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
3364                 return (error);
3365         umtxq_lock(&key);
3366         umtxq_busy(&key);
3367         cnt = umtxq_count(&key);
3368         if (cnt > 0) {
3369                 /*
3370                  * If this was the last sleeping thread, clear the waiters
3371                  * flag in _count.
3372                  */
3373                 if (cnt == 1) {
3374                         umtxq_unlock(&key);
3375                         rv = fueword32(&sem->_count, &count);
3376                         while (rv != -1 && count & USEM_HAS_WAITERS) {
3377                                 rv = casueword32(&sem->_count, count, &count,
3378                                     count & ~USEM_HAS_WAITERS);
3379                                 if (rv == 1) {
3380                                         rv = thread_check_susp(td, true);
3381                                         if (rv != 0)
3382                                                 break;
3383                                 }
3384                         }
3385                         if (rv == -1)
3386                                 error = EFAULT;
3387                         else if (rv > 0) {
3388                                 error = rv;
3389                         }
3390                         umtxq_lock(&key);
3391                 }
3392
3393                 umtxq_signal(&key, 1);
3394         }
3395         umtxq_unbusy(&key);
3396         umtxq_unlock(&key);
3397         umtx_key_release(&key);
3398         return (error);
3399 }
3400
3401 inline int
3402 umtx_copyin_timeout(const void *addr, struct timespec *tsp)
3403 {
3404         int error;
3405
3406         error = copyin(addr, tsp, sizeof(struct timespec));
3407         if (error == 0) {
3408                 if (tsp->tv_sec < 0 ||
3409                     tsp->tv_nsec >= 1000000000 ||
3410                     tsp->tv_nsec < 0)
3411                         error = EINVAL;
3412         }
3413         return (error);
3414 }
3415
3416 static inline int
3417 umtx_copyin_umtx_time(const void *addr, size_t size, struct _umtx_time *tp)
3418 {
3419         int error;
3420
3421         if (size <= sizeof(struct timespec)) {
3422                 tp->_clockid = CLOCK_REALTIME;
3423                 tp->_flags = 0;
3424                 error = copyin(addr, &tp->_timeout, sizeof(struct timespec));
3425         } else
3426                 error = copyin(addr, tp, sizeof(struct _umtx_time));
3427         if (error != 0)
3428                 return (error);
3429         if (tp->_timeout.tv_sec < 0 ||
3430             tp->_timeout.tv_nsec >= 1000000000 || tp->_timeout.tv_nsec < 0)
3431                 return (EINVAL);
3432         return (0);
3433 }
3434
3435 static int
3436 __umtx_op_unimpl(struct thread *td, struct _umtx_op_args *uap)
3437 {
3438
3439         return (EOPNOTSUPP);
3440 }
3441
3442 static int
3443 __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap)
3444 {
3445         struct _umtx_time timeout, *tm_p;
3446         int error;
3447
3448         if (uap->uaddr2 == NULL)
3449                 tm_p = NULL;
3450         else {
3451                 error = umtx_copyin_umtx_time(
3452                     uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3453                 if (error != 0)
3454                         return (error);
3455                 tm_p = &timeout;
3456         }
3457         return (do_wait(td, uap->obj, uap->val, tm_p, 0, 0));
3458 }
3459
3460 static int
3461 __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap)
3462 {
3463         struct _umtx_time timeout, *tm_p;
3464         int error;
3465
3466         if (uap->uaddr2 == NULL)
3467                 tm_p = NULL;
3468         else {
3469                 error = umtx_copyin_umtx_time(
3470                     uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3471                 if (error != 0)
3472                         return (error);
3473                 tm_p = &timeout;
3474         }
3475         return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0));
3476 }
3477
3478 static int
3479 __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap)
3480 {
3481         struct _umtx_time *tm_p, timeout;
3482         int error;
3483
3484         if (uap->uaddr2 == NULL)
3485                 tm_p = NULL;
3486         else {
3487                 error = umtx_copyin_umtx_time(
3488                     uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3489                 if (error != 0)
3490                         return (error);
3491                 tm_p = &timeout;
3492         }
3493         return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1));
3494 }
3495
3496 static int
3497 __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap)
3498 {
3499
3500         return (kern_umtx_wake(td, uap->obj, uap->val, 0));
3501 }
3502
3503 #define BATCH_SIZE      128
3504 static int
3505 __umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap)
3506 {
3507         char *uaddrs[BATCH_SIZE], **upp;
3508         int count, error, i, pos, tocopy;
3509
3510         upp = (char **)uap->obj;
3511         error = 0;
3512         for (count = uap->val, pos = 0; count > 0; count -= tocopy,
3513             pos += tocopy) {
3514                 tocopy = MIN(count, BATCH_SIZE);
3515                 error = copyin(upp + pos, uaddrs, tocopy * sizeof(char *));
3516                 if (error != 0)
3517                         break;
3518                 for (i = 0; i < tocopy; ++i)
3519                         kern_umtx_wake(td, uaddrs[i], INT_MAX, 1);
3520                 maybe_yield();
3521         }
3522         return (error);
3523 }
3524
3525 static int
3526 __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap)
3527 {
3528
3529         return (kern_umtx_wake(td, uap->obj, uap->val, 1));
3530 }
3531
3532 static int
3533 __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap)
3534 {
3535         struct _umtx_time *tm_p, timeout;
3536         int error;
3537
3538         /* Allow a null timespec (wait forever). */
3539         if (uap->uaddr2 == NULL)
3540                 tm_p = NULL;
3541         else {
3542                 error = umtx_copyin_umtx_time(
3543                     uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3544                 if (error != 0)
3545                         return (error);
3546                 tm_p = &timeout;
3547         }
3548         return (do_lock_umutex(td, uap->obj, tm_p, 0));
3549 }
3550
3551 static int
3552 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap)
3553 {
3554
3555         return (do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY));
3556 }
3557
3558 static int
3559 __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap)
3560 {
3561         struct _umtx_time *tm_p, timeout;
3562         int error;
3563
3564         /* Allow a null timespec (wait forever). */
3565         if (uap->uaddr2 == NULL)
3566                 tm_p = NULL;
3567         else {
3568                 error = umtx_copyin_umtx_time(
3569                     uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3570                 if (error != 0)
3571                         return (error);
3572                 tm_p = &timeout;
3573         }
3574         return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT));
3575 }
3576
3577 static int
3578 __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap)
3579 {
3580
3581         return (do_wake_umutex(td, uap->obj));
3582 }
3583
3584 static int
3585 __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap)
3586 {
3587
3588         return (do_unlock_umutex(td, uap->obj, false));
3589 }
3590
3591 static int
3592 __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap)
3593 {
3594
3595         return (do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1));
3596 }
3597
3598 static int
3599 __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap)
3600 {
3601         struct timespec *ts, timeout;
3602         int error;
3603
3604         /* Allow a null timespec (wait forever). */
3605         if (uap->uaddr2 == NULL)
3606                 ts = NULL;
3607         else {
3608                 error = umtx_copyin_timeout(uap->uaddr2, &timeout);
3609                 if (error != 0)
3610                         return (error);
3611                 ts = &timeout;
3612         }
3613         return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
3614 }
3615
3616 static int
3617 __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap)
3618 {
3619
3620         return (do_cv_signal(td, uap->obj));
3621 }
3622
3623 static int
3624 __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap)
3625 {
3626
3627         return (do_cv_broadcast(td, uap->obj));
3628 }
3629
3630 static int
3631 __umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap)
3632 {
3633         struct _umtx_time timeout;
3634         int error;
3635
3636         /* Allow a null timespec (wait forever). */
3637         if (uap->uaddr2 == NULL) {
3638                 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
3639         } else {
3640                 error = umtx_copyin_umtx_time(uap->uaddr2,
3641                    (size_t)uap->uaddr1, &timeout);
3642                 if (error != 0)
3643                         return (error);
3644                 error = do_rw_rdlock(td, uap->obj, uap->val, &timeout);
3645         }
3646         return (error);
3647 }
3648
3649 static int
3650 __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap)
3651 {
3652         struct _umtx_time timeout;
3653         int error;
3654
3655         /* Allow a null timespec (wait forever). */
3656         if (uap->uaddr2 == NULL) {
3657                 error = do_rw_wrlock(td, uap->obj, 0);
3658         } else {
3659                 error = umtx_copyin_umtx_time(uap->uaddr2,
3660                    (size_t)uap->uaddr1, &timeout);
3661                 if (error != 0)
3662                         return (error);
3663
3664                 error = do_rw_wrlock(td, uap->obj, &timeout);
3665         }
3666         return (error);
3667 }
3668
3669 static int
3670 __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap)
3671 {
3672
3673         return (do_rw_unlock(td, uap->obj));
3674 }
3675
3676 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
3677 static int
3678 __umtx_op_sem_wait(struct thread *td, struct _umtx_op_args *uap)
3679 {
3680         struct _umtx_time *tm_p, timeout;
3681         int error;
3682
3683         /* Allow a null timespec (wait forever). */
3684         if (uap->uaddr2 == NULL)
3685                 tm_p = NULL;
3686         else {
3687                 error = umtx_copyin_umtx_time(
3688                     uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3689                 if (error != 0)
3690                         return (error);
3691                 tm_p = &timeout;
3692         }
3693         return (do_sem_wait(td, uap->obj, tm_p));
3694 }
3695
3696 static int
3697 __umtx_op_sem_wake(struct thread *td, struct _umtx_op_args *uap)
3698 {
3699
3700         return (do_sem_wake(td, uap->obj));
3701 }
3702 #endif
3703
3704 static int
3705 __umtx_op_wake2_umutex(struct thread *td, struct _umtx_op_args *uap)
3706 {
3707
3708         return (do_wake2_umutex(td, uap->obj, uap->val));
3709 }
3710
3711 static int
3712 __umtx_op_sem2_wait(struct thread *td, struct _umtx_op_args *uap)
3713 {
3714         struct _umtx_time *tm_p, timeout;
3715         size_t uasize;
3716         int error;
3717
3718         /* Allow a null timespec (wait forever). */
3719         if (uap->uaddr2 == NULL) {
3720                 uasize = 0;
3721                 tm_p = NULL;
3722         } else {
3723                 uasize = (size_t)uap->uaddr1;
3724                 error = umtx_copyin_umtx_time(uap->uaddr2, uasize, &timeout);
3725                 if (error != 0)
3726                         return (error);
3727                 tm_p = &timeout;
3728         }
3729         error = do_sem2_wait(td, uap->obj, tm_p);
3730         if (error == EINTR && uap->uaddr2 != NULL &&
3731             (timeout._flags & UMTX_ABSTIME) == 0 &&
3732             uasize >= sizeof(struct _umtx_time) + sizeof(struct timespec)) {
3733                 error = copyout(&timeout._timeout,
3734                     (struct _umtx_time *)uap->uaddr2 + 1,
3735                     sizeof(struct timespec));
3736                 if (error == 0) {
3737                         error = EINTR;
3738                 }
3739         }
3740
3741         return (error);
3742 }
3743
3744 static int
3745 __umtx_op_sem2_wake(struct thread *td, struct _umtx_op_args *uap)
3746 {
3747
3748         return (do_sem2_wake(td, uap->obj));
3749 }
3750
3751 #define USHM_OBJ_UMTX(o)                                                \
3752     ((struct umtx_shm_obj_list *)(&(o)->umtx_data))
3753
3754 #define USHMF_REG_LINKED        0x0001
3755 #define USHMF_OBJ_LINKED        0x0002
3756 struct umtx_shm_reg {
3757         TAILQ_ENTRY(umtx_shm_reg) ushm_reg_link;
3758         LIST_ENTRY(umtx_shm_reg) ushm_obj_link;
3759         struct umtx_key         ushm_key;
3760         struct ucred            *ushm_cred;
3761         struct shmfd            *ushm_obj;
3762         u_int                   ushm_refcnt;
3763         u_int                   ushm_flags;
3764 };
3765
3766 LIST_HEAD(umtx_shm_obj_list, umtx_shm_reg);
3767 TAILQ_HEAD(umtx_shm_reg_head, umtx_shm_reg);
3768
3769 static uma_zone_t umtx_shm_reg_zone;
3770 static struct umtx_shm_reg_head umtx_shm_registry[UMTX_CHAINS];
3771 static struct mtx umtx_shm_lock;
3772 static struct umtx_shm_reg_head umtx_shm_reg_delfree =
3773     TAILQ_HEAD_INITIALIZER(umtx_shm_reg_delfree);
3774
3775 static void umtx_shm_free_reg(struct umtx_shm_reg *reg);
3776
3777 static void
3778 umtx_shm_reg_delfree_tq(void *context __unused, int pending __unused)
3779 {
3780         struct umtx_shm_reg_head d;
3781         struct umtx_shm_reg *reg, *reg1;
3782
3783         TAILQ_INIT(&d);
3784         mtx_lock(&umtx_shm_lock);
3785         TAILQ_CONCAT(&d, &umtx_shm_reg_delfree, ushm_reg_link);
3786         mtx_unlock(&umtx_shm_lock);
3787         TAILQ_FOREACH_SAFE(reg, &d, ushm_reg_link, reg1) {
3788                 TAILQ_REMOVE(&d, reg, ushm_reg_link);
3789                 umtx_shm_free_reg(reg);
3790         }
3791 }
3792
3793 static struct task umtx_shm_reg_delfree_task =
3794     TASK_INITIALIZER(0, umtx_shm_reg_delfree_tq, NULL);
3795
3796 static struct umtx_shm_reg *
3797 umtx_shm_find_reg_locked(const struct umtx_key *key)
3798 {
3799         struct umtx_shm_reg *reg;
3800         struct umtx_shm_reg_head *reg_head;
3801
3802         KASSERT(key->shared, ("umtx_p_find_rg: private key"));
3803         mtx_assert(&umtx_shm_lock, MA_OWNED);
3804         reg_head = &umtx_shm_registry[key->hash];
3805         TAILQ_FOREACH(reg, reg_head, ushm_reg_link) {
3806                 KASSERT(reg->ushm_key.shared,
3807                     ("non-shared key on reg %p %d", reg, reg->ushm_key.shared));
3808                 if (reg->ushm_key.info.shared.object ==
3809                     key->info.shared.object &&
3810                     reg->ushm_key.info.shared.offset ==
3811                     key->info.shared.offset) {
3812                         KASSERT(reg->ushm_key.type == TYPE_SHM, ("TYPE_USHM"));
3813                         KASSERT(reg->ushm_refcnt > 0,
3814                             ("reg %p refcnt 0 onlist", reg));
3815                         KASSERT((reg->ushm_flags & USHMF_REG_LINKED) != 0,
3816                             ("reg %p not linked", reg));
3817                         reg->ushm_refcnt++;
3818                         return (reg);
3819                 }
3820         }
3821         return (NULL);
3822 }
3823
3824 static struct umtx_shm_reg *
3825 umtx_shm_find_reg(const struct umtx_key *key)
3826 {
3827         struct umtx_shm_reg *reg;
3828
3829         mtx_lock(&umtx_shm_lock);
3830         reg = umtx_shm_find_reg_locked(key);
3831         mtx_unlock(&umtx_shm_lock);
3832         return (reg);
3833 }
3834
3835 static void
3836 umtx_shm_free_reg(struct umtx_shm_reg *reg)
3837 {
3838
3839         chgumtxcnt(reg->ushm_cred->cr_ruidinfo, -1, 0);
3840         crfree(reg->ushm_cred);
3841         shm_drop(reg->ushm_obj);
3842         uma_zfree(umtx_shm_reg_zone, reg);
3843 }
3844
3845 static bool
3846 umtx_shm_unref_reg_locked(struct umtx_shm_reg *reg, bool force)
3847 {
3848         bool res;
3849
3850         mtx_assert(&umtx_shm_lock, MA_OWNED);
3851         KASSERT(reg->ushm_refcnt > 0, ("ushm_reg %p refcnt 0", reg));
3852         reg->ushm_refcnt--;
3853         res = reg->ushm_refcnt == 0;
3854         if (res || force) {
3855                 if ((reg->ushm_flags & USHMF_REG_LINKED) != 0) {
3856                         TAILQ_REMOVE(&umtx_shm_registry[reg->ushm_key.hash],
3857                             reg, ushm_reg_link);
3858                         reg->ushm_flags &= ~USHMF_REG_LINKED;
3859                 }
3860                 if ((reg->ushm_flags & USHMF_OBJ_LINKED) != 0) {
3861                         LIST_REMOVE(reg, ushm_obj_link);
3862                         reg->ushm_flags &= ~USHMF_OBJ_LINKED;
3863                 }
3864         }
3865         return (res);
3866 }
3867
3868 static void
3869 umtx_shm_unref_reg(struct umtx_shm_reg *reg, bool force)
3870 {
3871         vm_object_t object;
3872         bool dofree;
3873
3874         if (force) {
3875                 object = reg->ushm_obj->shm_object;
3876                 VM_OBJECT_WLOCK(object);
3877                 object->flags |= OBJ_UMTXDEAD;
3878                 VM_OBJECT_WUNLOCK(object);
3879         }
3880         mtx_lock(&umtx_shm_lock);
3881         dofree = umtx_shm_unref_reg_locked(reg, force);
3882         mtx_unlock(&umtx_shm_lock);
3883         if (dofree)
3884                 umtx_shm_free_reg(reg);
3885 }
3886
3887 void
3888 umtx_shm_object_init(vm_object_t object)
3889 {
3890
3891         LIST_INIT(USHM_OBJ_UMTX(object));
3892 }
3893
3894 void
3895 umtx_shm_object_terminated(vm_object_t object)
3896 {
3897         struct umtx_shm_reg *reg, *reg1;
3898         bool dofree;
3899
3900         if (LIST_EMPTY(USHM_OBJ_UMTX(object)))
3901                 return;
3902
3903         dofree = false;
3904         mtx_lock(&umtx_shm_lock);
3905         LIST_FOREACH_SAFE(reg, USHM_OBJ_UMTX(object), ushm_obj_link, reg1) {
3906                 if (umtx_shm_unref_reg_locked(reg, true)) {
3907                         TAILQ_INSERT_TAIL(&umtx_shm_reg_delfree, reg,
3908                             ushm_reg_link);
3909                         dofree = true;
3910                 }
3911         }
3912         mtx_unlock(&umtx_shm_lock);
3913         if (dofree)
3914                 taskqueue_enqueue(taskqueue_thread, &umtx_shm_reg_delfree_task);
3915 }
3916
3917 static int
3918 umtx_shm_create_reg(struct thread *td, const struct umtx_key *key,
3919     struct umtx_shm_reg **res)
3920 {
3921         struct umtx_shm_reg *reg, *reg1;
3922         struct ucred *cred;
3923         int error;
3924
3925         reg = umtx_shm_find_reg(key);
3926         if (reg != NULL) {
3927                 *res = reg;
3928                 return (0);
3929         }
3930         cred = td->td_ucred;
3931         if (!chgumtxcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_UMTXP)))
3932                 return (ENOMEM);
3933         reg = uma_zalloc(umtx_shm_reg_zone, M_WAITOK | M_ZERO);
3934         reg->ushm_refcnt = 1;
3935         bcopy(key, &reg->ushm_key, sizeof(*key));
3936         reg->ushm_obj = shm_alloc(td->td_ucred, O_RDWR);
3937         reg->ushm_cred = crhold(cred);
3938         error = shm_dotruncate(reg->ushm_obj, PAGE_SIZE);
3939         if (error != 0) {
3940                 umtx_shm_free_reg(reg);
3941                 return (error);
3942         }
3943         mtx_lock(&umtx_shm_lock);
3944         reg1 = umtx_shm_find_reg_locked(key);
3945         if (reg1 != NULL) {
3946                 mtx_unlock(&umtx_shm_lock);
3947                 umtx_shm_free_reg(reg);
3948                 *res = reg1;
3949                 return (0);
3950         }
3951         reg->ushm_refcnt++;
3952         TAILQ_INSERT_TAIL(&umtx_shm_registry[key->hash], reg, ushm_reg_link);
3953         LIST_INSERT_HEAD(USHM_OBJ_UMTX(key->info.shared.object), reg,
3954             ushm_obj_link);
3955         reg->ushm_flags = USHMF_REG_LINKED | USHMF_OBJ_LINKED;
3956         mtx_unlock(&umtx_shm_lock);
3957         *res = reg;
3958         return (0);
3959 }
3960
3961 static int
3962 umtx_shm_alive(struct thread *td, void *addr)
3963 {
3964         vm_map_t map;
3965         vm_map_entry_t entry;
3966         vm_object_t object;
3967         vm_pindex_t pindex;
3968         vm_prot_t prot;
3969         int res, ret;
3970         boolean_t wired;
3971
3972         map = &td->td_proc->p_vmspace->vm_map;
3973         res = vm_map_lookup(&map, (uintptr_t)addr, VM_PROT_READ, &entry,
3974             &object, &pindex, &prot, &wired);
3975         if (res != KERN_SUCCESS)
3976                 return (EFAULT);
3977         if (object == NULL)
3978                 ret = EINVAL;
3979         else
3980                 ret = (object->flags & OBJ_UMTXDEAD) != 0 ? ENOTTY : 0;
3981         vm_map_lookup_done(map, entry);
3982         return (ret);
3983 }
3984
3985 static void
3986 umtx_shm_init(void)
3987 {
3988         int i;
3989
3990         umtx_shm_reg_zone = uma_zcreate("umtx_shm", sizeof(struct umtx_shm_reg),
3991             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
3992         mtx_init(&umtx_shm_lock, "umtxshm", NULL, MTX_DEF);
3993         for (i = 0; i < nitems(umtx_shm_registry); i++)
3994                 TAILQ_INIT(&umtx_shm_registry[i]);
3995 }
3996
3997 static int
3998 umtx_shm(struct thread *td, void *addr, u_int flags)
3999 {
4000         struct umtx_key key;
4001         struct umtx_shm_reg *reg;
4002         struct file *fp;
4003         int error, fd;
4004
4005         if (__bitcount(flags & (UMTX_SHM_CREAT | UMTX_SHM_LOOKUP |
4006             UMTX_SHM_DESTROY| UMTX_SHM_ALIVE)) != 1)
4007                 return (EINVAL);
4008         if ((flags & UMTX_SHM_ALIVE) != 0)
4009                 return (umtx_shm_alive(td, addr));
4010         error = umtx_key_get(addr, TYPE_SHM, PROCESS_SHARE, &key);
4011         if (error != 0)
4012                 return (error);
4013         KASSERT(key.shared == 1, ("non-shared key"));
4014         if ((flags & UMTX_SHM_CREAT) != 0) {
4015                 error = umtx_shm_create_reg(td, &key, &reg);
4016         } else {
4017                 reg = umtx_shm_find_reg(&key);
4018                 if (reg == NULL)
4019                         error = ESRCH;
4020         }
4021         umtx_key_release(&key);
4022         if (error != 0)
4023                 return (error);
4024         KASSERT(reg != NULL, ("no reg"));
4025         if ((flags & UMTX_SHM_DESTROY) != 0) {
4026                 umtx_shm_unref_reg(reg, true);
4027         } else {
4028 #if 0
4029 #ifdef MAC
4030                 error = mac_posixshm_check_open(td->td_ucred,
4031                     reg->ushm_obj, FFLAGS(O_RDWR));
4032                 if (error == 0)
4033 #endif
4034                         error = shm_access(reg->ushm_obj, td->td_ucred,
4035                             FFLAGS(O_RDWR));
4036                 if (error == 0)
4037 #endif
4038                         error = falloc_caps(td, &fp, &fd, O_CLOEXEC, NULL);
4039                 if (error == 0) {
4040                         shm_hold(reg->ushm_obj);
4041                         finit(fp, FFLAGS(O_RDWR), DTYPE_SHM, reg->ushm_obj,
4042                             &shm_ops);
4043                         td->td_retval[0] = fd;
4044                         fdrop(fp, td);
4045                 }
4046         }
4047         umtx_shm_unref_reg(reg, false);
4048         return (error);
4049 }
4050
4051 static int
4052 __umtx_op_shm(struct thread *td, struct _umtx_op_args *uap)
4053 {
4054
4055         return (umtx_shm(td, uap->uaddr1, uap->val));
4056 }
4057
4058 static int
4059 umtx_robust_lists(struct thread *td, struct umtx_robust_lists_params *rbp)
4060 {
4061
4062         td->td_rb_list = rbp->robust_list_offset;
4063         td->td_rbp_list = rbp->robust_priv_list_offset;
4064         td->td_rb_inact = rbp->robust_inact_offset;
4065         return (0);
4066 }
4067
4068 static int
4069 __umtx_op_robust_lists(struct thread *td, struct _umtx_op_args *uap)
4070 {
4071         struct umtx_robust_lists_params rb;
4072         int error;
4073
4074         if (uap->val > sizeof(rb))
4075                 return (EINVAL);
4076         bzero(&rb, sizeof(rb));
4077         error = copyin(uap->uaddr1, &rb, uap->val);
4078         if (error != 0)
4079                 return (error);
4080         return (umtx_robust_lists(td, &rb));
4081 }
4082
4083 typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap);
4084
4085 static const _umtx_op_func op_table[] = {
4086         [UMTX_OP_RESERVED0]     = __umtx_op_unimpl,
4087         [UMTX_OP_RESERVED1]     = __umtx_op_unimpl,
4088         [UMTX_OP_WAIT]          = __umtx_op_wait,
4089         [UMTX_OP_WAKE]          = __umtx_op_wake,
4090         [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex,
4091         [UMTX_OP_MUTEX_LOCK]    = __umtx_op_lock_umutex,
4092         [UMTX_OP_MUTEX_UNLOCK]  = __umtx_op_unlock_umutex,
4093         [UMTX_OP_SET_CEILING]   = __umtx_op_set_ceiling,
4094         [UMTX_OP_CV_WAIT]       = __umtx_op_cv_wait,
4095         [UMTX_OP_CV_SIGNAL]     = __umtx_op_cv_signal,
4096         [UMTX_OP_CV_BROADCAST]  = __umtx_op_cv_broadcast,
4097         [UMTX_OP_WAIT_UINT]     = __umtx_op_wait_uint,
4098         [UMTX_OP_RW_RDLOCK]     = __umtx_op_rw_rdlock,
4099         [UMTX_OP_RW_WRLOCK]     = __umtx_op_rw_wrlock,
4100         [UMTX_OP_RW_UNLOCK]     = __umtx_op_rw_unlock,
4101         [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private,
4102         [UMTX_OP_WAKE_PRIVATE]  = __umtx_op_wake_private,
4103         [UMTX_OP_MUTEX_WAIT]    = __umtx_op_wait_umutex,
4104         [UMTX_OP_MUTEX_WAKE]    = __umtx_op_wake_umutex,
4105 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
4106         [UMTX_OP_SEM_WAIT]      = __umtx_op_sem_wait,
4107         [UMTX_OP_SEM_WAKE]      = __umtx_op_sem_wake,
4108 #else
4109         [UMTX_OP_SEM_WAIT]      = __umtx_op_unimpl,
4110         [UMTX_OP_SEM_WAKE]      = __umtx_op_unimpl,
4111 #endif
4112         [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private,
4113         [UMTX_OP_MUTEX_WAKE2]   = __umtx_op_wake2_umutex,
4114         [UMTX_OP_SEM2_WAIT]     = __umtx_op_sem2_wait,
4115         [UMTX_OP_SEM2_WAKE]     = __umtx_op_sem2_wake,
4116         [UMTX_OP_SHM]           = __umtx_op_shm,
4117         [UMTX_OP_ROBUST_LISTS]  = __umtx_op_robust_lists,
4118 };
4119
4120 int
4121 sys__umtx_op(struct thread *td, struct _umtx_op_args *uap)
4122 {
4123
4124         if ((unsigned)uap->op < nitems(op_table))
4125                 return (*op_table[uap->op])(td, uap);
4126         return (EINVAL);
4127 }
4128
4129 #ifdef COMPAT_FREEBSD32
4130
4131 struct timespec32 {
4132         int32_t tv_sec;
4133         int32_t tv_nsec;
4134 };
4135
4136 struct umtx_time32 {
4137         struct  timespec32      timeout;
4138         uint32_t                flags;
4139         uint32_t                clockid;
4140 };
4141
4142 static inline int
4143 umtx_copyin_timeout32(void *addr, struct timespec *tsp)
4144 {
4145         struct timespec32 ts32;
4146         int error;
4147
4148         error = copyin(addr, &ts32, sizeof(struct timespec32));
4149         if (error == 0) {
4150                 if (ts32.tv_sec < 0 ||
4151                     ts32.tv_nsec >= 1000000000 ||
4152                     ts32.tv_nsec < 0)
4153                         error = EINVAL;
4154                 else {
4155                         tsp->tv_sec = ts32.tv_sec;
4156                         tsp->tv_nsec = ts32.tv_nsec;
4157                 }
4158         }
4159         return (error);
4160 }
4161
4162 static inline int
4163 umtx_copyin_umtx_time32(const void *addr, size_t size, struct _umtx_time *tp)
4164 {
4165         struct umtx_time32 t32;
4166         int error;
4167
4168         t32.clockid = CLOCK_REALTIME;
4169         t32.flags   = 0;
4170         if (size <= sizeof(struct timespec32))
4171                 error = copyin(addr, &t32.timeout, sizeof(struct timespec32));
4172         else
4173                 error = copyin(addr, &t32, sizeof(struct umtx_time32));
4174         if (error != 0)
4175                 return (error);
4176         if (t32.timeout.tv_sec < 0 ||
4177             t32.timeout.tv_nsec >= 1000000000 || t32.timeout.tv_nsec < 0)
4178                 return (EINVAL);
4179         tp->_timeout.tv_sec = t32.timeout.tv_sec;
4180         tp->_timeout.tv_nsec = t32.timeout.tv_nsec;
4181         tp->_flags = t32.flags;
4182         tp->_clockid = t32.clockid;
4183         return (0);
4184 }
4185
4186 static int
4187 __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
4188 {
4189         struct _umtx_time *tm_p, timeout;
4190         int error;
4191
4192         if (uap->uaddr2 == NULL)
4193                 tm_p = NULL;
4194         else {
4195                 error = umtx_copyin_umtx_time32(uap->uaddr2,
4196                         (size_t)uap->uaddr1, &timeout);
4197                 if (error != 0)
4198                         return (error);
4199                 tm_p = &timeout;
4200         }
4201         return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0));
4202 }
4203
4204 static int
4205 __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
4206 {
4207         struct _umtx_time *tm_p, timeout;
4208         int error;
4209
4210         /* Allow a null timespec (wait forever). */
4211         if (uap->uaddr2 == NULL)
4212                 tm_p = NULL;
4213         else {
4214                 error = umtx_copyin_umtx_time32(uap->uaddr2,
4215                             (size_t)uap->uaddr1, &timeout);
4216                 if (error != 0)
4217                         return (error);
4218                 tm_p = &timeout;
4219         }
4220         return (do_lock_umutex(td, uap->obj, tm_p, 0));
4221 }
4222
4223 static int
4224 __umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
4225 {
4226         struct _umtx_time *tm_p, timeout;
4227         int error;
4228
4229         /* Allow a null timespec (wait forever). */
4230         if (uap->uaddr2 == NULL)
4231                 tm_p = NULL;
4232         else {
4233                 error = umtx_copyin_umtx_time32(uap->uaddr2,
4234                     (size_t)uap->uaddr1, &timeout);
4235                 if (error != 0)
4236                         return (error);
4237                 tm_p = &timeout;
4238         }
4239         return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT));
4240 }
4241
4242 static int
4243 __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
4244 {
4245         struct timespec *ts, timeout;
4246         int error;
4247
4248         /* Allow a null timespec (wait forever). */
4249         if (uap->uaddr2 == NULL)
4250                 ts = NULL;
4251         else {
4252                 error = umtx_copyin_timeout32(uap->uaddr2, &timeout);
4253                 if (error != 0)
4254                         return (error);
4255                 ts = &timeout;
4256         }
4257         return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
4258 }
4259
4260 static int
4261 __umtx_op_rw_rdlock_compat32(struct thread *td, struct _umtx_op_args *uap)
4262 {
4263         struct _umtx_time timeout;
4264         int error;
4265
4266         /* Allow a null timespec (wait forever). */
4267         if (uap->uaddr2 == NULL) {
4268                 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
4269         } else {
4270                 error = umtx_copyin_umtx_time32(uap->uaddr2,
4271                     (size_t)uap->uaddr1, &timeout);
4272                 if (error != 0)
4273                         return (error);
4274                 error = do_rw_rdlock(td, uap->obj, uap->val, &timeout);
4275         }
4276         return (error);
4277 }
4278
4279 static int
4280 __umtx_op_rw_wrlock_compat32(struct thread *td, struct _umtx_op_args *uap)
4281 {
4282         struct _umtx_time timeout;
4283         int error;
4284
4285         /* Allow a null timespec (wait forever). */
4286         if (uap->uaddr2 == NULL) {
4287                 error = do_rw_wrlock(td, uap->obj, 0);
4288         } else {
4289                 error = umtx_copyin_umtx_time32(uap->uaddr2,
4290                     (size_t)uap->uaddr1, &timeout);
4291                 if (error != 0)
4292                         return (error);
4293                 error = do_rw_wrlock(td, uap->obj, &timeout);
4294         }
4295         return (error);
4296 }
4297
4298 static int
4299 __umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *uap)
4300 {
4301         struct _umtx_time *tm_p, timeout;
4302         int error;
4303
4304         if (uap->uaddr2 == NULL)
4305                 tm_p = NULL;
4306         else {
4307                 error = umtx_copyin_umtx_time32(
4308                     uap->uaddr2, (size_t)uap->uaddr1,&timeout);
4309                 if (error != 0)
4310                         return (error);
4311                 tm_p = &timeout;
4312         }
4313         return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1));
4314 }
4315
4316 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
4317 static int
4318 __umtx_op_sem_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
4319 {
4320         struct _umtx_time *tm_p, timeout;
4321         int error;
4322
4323         /* Allow a null timespec (wait forever). */
4324         if (uap->uaddr2 == NULL)
4325                 tm_p = NULL;
4326         else {
4327                 error = umtx_copyin_umtx_time32(uap->uaddr2,
4328                     (size_t)uap->uaddr1, &timeout);
4329                 if (error != 0)
4330                         return (error);
4331                 tm_p = &timeout;
4332         }
4333         return (do_sem_wait(td, uap->obj, tm_p));
4334 }
4335 #endif
4336
4337 static int
4338 __umtx_op_sem2_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
4339 {
4340         struct _umtx_time *tm_p, timeout;
4341         size_t uasize;
4342         int error;
4343
4344         /* Allow a null timespec (wait forever). */
4345         if (uap->uaddr2 == NULL) {
4346                 uasize = 0;
4347                 tm_p = NULL;
4348         } else {
4349                 uasize = (size_t)uap->uaddr1;
4350                 error = umtx_copyin_umtx_time32(uap->uaddr2, uasize, &timeout);
4351                 if (error != 0)
4352                         return (error);
4353                 tm_p = &timeout;
4354         }
4355         error = do_sem2_wait(td, uap->obj, tm_p);
4356         if (error == EINTR && uap->uaddr2 != NULL &&
4357             (timeout._flags & UMTX_ABSTIME) == 0 &&
4358             uasize >= sizeof(struct umtx_time32) + sizeof(struct timespec32)) {
4359                 struct timespec32 remain32 = {
4360                         .tv_sec = timeout._timeout.tv_sec,
4361                         .tv_nsec = timeout._timeout.tv_nsec
4362                 };
4363                 error = copyout(&remain32,
4364                     (struct umtx_time32 *)uap->uaddr2 + 1,
4365                     sizeof(struct timespec32));
4366                 if (error == 0) {
4367                         error = EINTR;
4368                 }
4369         }
4370
4371         return (error);
4372 }
4373
4374 static int
4375 __umtx_op_nwake_private32(struct thread *td, struct _umtx_op_args *uap)
4376 {
4377         uint32_t uaddrs[BATCH_SIZE], **upp;
4378         int count, error, i, pos, tocopy;
4379
4380         upp = (uint32_t **)uap->obj;
4381         error = 0;
4382         for (count = uap->val, pos = 0; count > 0; count -= tocopy,
4383             pos += tocopy) {
4384                 tocopy = MIN(count, BATCH_SIZE);
4385                 error = copyin(upp + pos, uaddrs, tocopy * sizeof(uint32_t));
4386                 if (error != 0)
4387                         break;
4388                 for (i = 0; i < tocopy; ++i)
4389                         kern_umtx_wake(td, (void *)(intptr_t)uaddrs[i],
4390                             INT_MAX, 1);
4391                 maybe_yield();
4392         }
4393         return (error);
4394 }
4395
4396 struct umtx_robust_lists_params_compat32 {
4397         uint32_t        robust_list_offset;
4398         uint32_t        robust_priv_list_offset;
4399         uint32_t        robust_inact_offset;
4400 };
4401
4402 static int
4403 __umtx_op_robust_lists_compat32(struct thread *td, struct _umtx_op_args *uap)
4404 {
4405         struct umtx_robust_lists_params rb;
4406         struct umtx_robust_lists_params_compat32 rb32;
4407         int error;
4408
4409         if (uap->val > sizeof(rb32))
4410                 return (EINVAL);
4411         bzero(&rb, sizeof(rb));
4412         bzero(&rb32, sizeof(rb32));
4413         error = copyin(uap->uaddr1, &rb32, uap->val);
4414         if (error != 0)
4415                 return (error);
4416         rb.robust_list_offset = rb32.robust_list_offset;
4417         rb.robust_priv_list_offset = rb32.robust_priv_list_offset;
4418         rb.robust_inact_offset = rb32.robust_inact_offset;
4419         return (umtx_robust_lists(td, &rb));
4420 }
4421
4422 static const _umtx_op_func op_table_compat32[] = {
4423         [UMTX_OP_RESERVED0]     = __umtx_op_unimpl,
4424         [UMTX_OP_RESERVED1]     = __umtx_op_unimpl,
4425         [UMTX_OP_WAIT]          = __umtx_op_wait_compat32,
4426         [UMTX_OP_WAKE]          = __umtx_op_wake,
4427         [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex,
4428         [UMTX_OP_MUTEX_LOCK]    = __umtx_op_lock_umutex_compat32,
4429         [UMTX_OP_MUTEX_UNLOCK]  = __umtx_op_unlock_umutex,
4430         [UMTX_OP_SET_CEILING]   = __umtx_op_set_ceiling,
4431         [UMTX_OP_CV_WAIT]       = __umtx_op_cv_wait_compat32,
4432         [UMTX_OP_CV_SIGNAL]     = __umtx_op_cv_signal,
4433         [UMTX_OP_CV_BROADCAST]  = __umtx_op_cv_broadcast,
4434         [UMTX_OP_WAIT_UINT]     = __umtx_op_wait_compat32,
4435         [UMTX_OP_RW_RDLOCK]     = __umtx_op_rw_rdlock_compat32,
4436         [UMTX_OP_RW_WRLOCK]     = __umtx_op_rw_wrlock_compat32,
4437         [UMTX_OP_RW_UNLOCK]     = __umtx_op_rw_unlock,
4438         [UMTX_OP_WAIT_UINT_PRIVATE] = __umtx_op_wait_uint_private_compat32,
4439         [UMTX_OP_WAKE_PRIVATE]  = __umtx_op_wake_private,
4440         [UMTX_OP_MUTEX_WAIT]    = __umtx_op_wait_umutex_compat32,
4441         [UMTX_OP_MUTEX_WAKE]    = __umtx_op_wake_umutex,
4442 #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10)
4443         [UMTX_OP_SEM_WAIT]      = __umtx_op_sem_wait_compat32,
4444         [UMTX_OP_SEM_WAKE]      = __umtx_op_sem_wake,
4445 #else
4446         [UMTX_OP_SEM_WAIT]      = __umtx_op_unimpl,
4447         [UMTX_OP_SEM_WAKE]      = __umtx_op_unimpl,
4448 #endif
4449         [UMTX_OP_NWAKE_PRIVATE] = __umtx_op_nwake_private32,
4450         [UMTX_OP_MUTEX_WAKE2]   = __umtx_op_wake2_umutex,
4451         [UMTX_OP_SEM2_WAIT]     = __umtx_op_sem2_wait_compat32,
4452         [UMTX_OP_SEM2_WAKE]     = __umtx_op_sem2_wake,
4453         [UMTX_OP_SHM]           = __umtx_op_shm,
4454         [UMTX_OP_ROBUST_LISTS]  = __umtx_op_robust_lists_compat32,
4455 };
4456
4457 int
4458 freebsd32__umtx_op(struct thread *td, struct freebsd32__umtx_op_args *uap)
4459 {
4460
4461         if ((unsigned)uap->op < nitems(op_table_compat32)) {
4462                 return (*op_table_compat32[uap->op])(td,
4463                     (struct _umtx_op_args *)uap);
4464         }
4465         return (EINVAL);
4466 }
4467 #endif
4468
4469 void
4470 umtx_thread_init(struct thread *td)
4471 {
4472
4473         td->td_umtxq = umtxq_alloc();
4474         td->td_umtxq->uq_thread = td;
4475 }
4476
4477 void
4478 umtx_thread_fini(struct thread *td)
4479 {
4480
4481         umtxq_free(td->td_umtxq);
4482 }
4483
4484 /*
4485  * It will be called when new thread is created, e.g fork().
4486  */
4487 void
4488 umtx_thread_alloc(struct thread *td)
4489 {
4490         struct umtx_q *uq;
4491
4492         uq = td->td_umtxq;
4493         uq->uq_inherited_pri = PRI_MAX;
4494
4495         KASSERT(uq->uq_flags == 0, ("uq_flags != 0"));
4496         KASSERT(uq->uq_thread == td, ("uq_thread != td"));
4497         KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL"));
4498         KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty"));
4499 }
4500
4501 /*
4502  * exec() hook.
4503  *
4504  * Clear robust lists for all process' threads, not delaying the
4505  * cleanup to thread_exit hook, since the relevant address space is
4506  * destroyed right now.
4507  */
4508 static void
4509 umtx_exec_hook(void *arg __unused, struct proc *p,
4510     struct image_params *imgp __unused)
4511 {
4512         struct thread *td;
4513
4514         KASSERT(p == curproc, ("need curproc"));
4515         KASSERT((p->p_flag & P_HADTHREADS) == 0 ||
4516             (p->p_flag & P_STOPPED_SINGLE) != 0,
4517             ("curproc must be single-threaded"));
4518         /*
4519          * There is no need to lock the list as only this thread can be
4520          * running.
4521          */
4522         FOREACH_THREAD_IN_PROC(p, td) {
4523                 KASSERT(td == curthread ||
4524                     ((td->td_flags & TDF_BOUNDARY) != 0 && TD_IS_SUSPENDED(td)),
4525                     ("running thread %p %p", p, td));
4526                 umtx_thread_cleanup(td);
4527                 td->td_rb_list = td->td_rbp_list = td->td_rb_inact = 0;
4528         }
4529 }
4530
4531 /*
4532  * thread_exit() hook.
4533  */
4534 void
4535 umtx_thread_exit(struct thread *td)
4536 {
4537
4538         umtx_thread_cleanup(td);
4539 }
4540
4541 static int
4542 umtx_read_uptr(struct thread *td, uintptr_t ptr, uintptr_t *res)
4543 {
4544         u_long res1;
4545 #ifdef COMPAT_FREEBSD32
4546         uint32_t res32;
4547 #endif
4548         int error;
4549
4550 #ifdef COMPAT_FREEBSD32
4551         if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
4552                 error = fueword32((void *)ptr, &res32);
4553                 if (error == 0)
4554                         res1 = res32;
4555         } else
4556 #endif
4557         {
4558                 error = fueword((void *)ptr, &res1);
4559         }
4560         if (error == 0)
4561                 *res = res1;
4562         else
4563                 error = EFAULT;
4564         return (error);
4565 }
4566
4567 static void
4568 umtx_read_rb_list(struct thread *td, struct umutex *m, uintptr_t *rb_list)
4569 {
4570 #ifdef COMPAT_FREEBSD32
4571         struct umutex32 m32;
4572
4573         if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
4574                 memcpy(&m32, m, sizeof(m32));
4575                 *rb_list = m32.m_rb_lnk;
4576         } else
4577 #endif
4578                 *rb_list = m->m_rb_lnk;
4579 }
4580
4581 static int
4582 umtx_handle_rb(struct thread *td, uintptr_t rbp, uintptr_t *rb_list, bool inact)
4583 {
4584         struct umutex m;
4585         int error;
4586
4587         KASSERT(td->td_proc == curproc, ("need current vmspace"));
4588         error = copyin((void *)rbp, &m, sizeof(m));
4589         if (error != 0)
4590                 return (error);
4591         if (rb_list != NULL)
4592                 umtx_read_rb_list(td, &m, rb_list);
4593         if ((m.m_flags & UMUTEX_ROBUST) == 0)
4594                 return (EINVAL);
4595         if ((m.m_owner & ~UMUTEX_CONTESTED) != td->td_tid)
4596                 /* inact is cleared after unlock, allow the inconsistency */
4597                 return (inact ? 0 : EINVAL);
4598         return (do_unlock_umutex(td, (struct umutex *)rbp, true));
4599 }
4600
4601 static void
4602 umtx_cleanup_rb_list(struct thread *td, uintptr_t rb_list, uintptr_t *rb_inact,
4603     const char *name)
4604 {
4605         int error, i;
4606         uintptr_t rbp;
4607         bool inact;
4608
4609         if (rb_list == 0)
4610                 return;
4611         error = umtx_read_uptr(td, rb_list, &rbp);
4612         for (i = 0; error == 0 && rbp != 0 && i < umtx_max_rb; i++) {
4613                 if (rbp == *rb_inact) {
4614                         inact = true;
4615                         *rb_inact = 0;
4616                 } else
4617                         inact = false;
4618                 error = umtx_handle_rb(td, rbp, &rbp, inact);
4619         }
4620         if (i == umtx_max_rb && umtx_verbose_rb) {
4621                 uprintf("comm %s pid %d: reached umtx %smax rb %d\n",
4622                     td->td_proc->p_comm, td->td_proc->p_pid, name, umtx_max_rb);
4623         }
4624         if (error != 0 && umtx_verbose_rb) {
4625                 uprintf("comm %s pid %d: handling %srb error %d\n",
4626                     td->td_proc->p_comm, td->td_proc->p_pid, name, error);
4627         }
4628 }
4629
4630 /*
4631  * Clean up umtx data.
4632  */
4633 static void
4634 umtx_thread_cleanup(struct thread *td)
4635 {
4636         struct umtx_q *uq;
4637         struct umtx_pi *pi;
4638         uintptr_t rb_inact;
4639
4640         /*
4641          * Disown pi mutexes.
4642          */
4643         uq = td->td_umtxq;
4644         if (uq != NULL) {
4645                 if (uq->uq_inherited_pri != PRI_MAX ||
4646                     !TAILQ_EMPTY(&uq->uq_pi_contested)) {
4647                         mtx_lock(&umtx_lock);
4648                         uq->uq_inherited_pri = PRI_MAX;
4649                         while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) {
4650                                 pi->pi_owner = NULL;
4651                                 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);
4652                         }
4653                         mtx_unlock(&umtx_lock);
4654                 }
4655                 sched_lend_user_prio_cond(td, PRI_MAX);
4656         }
4657
4658         if (td->td_rb_inact == 0 && td->td_rb_list == 0 && td->td_rbp_list == 0)
4659                 return;
4660
4661         /*
4662          * Handle terminated robust mutexes.  Must be done after
4663          * robust pi disown, otherwise unlock could see unowned
4664          * entries.
4665          */
4666         rb_inact = td->td_rb_inact;
4667         if (rb_inact != 0)
4668                 (void)umtx_read_uptr(td, rb_inact, &rb_inact);
4669         umtx_cleanup_rb_list(td, td->td_rb_list, &rb_inact, "");
4670         umtx_cleanup_rb_list(td, td->td_rbp_list, &rb_inact, "priv ");
4671         if (rb_inact != 0)
4672                 (void)umtx_handle_rb(td, rb_inact, NULL, true);
4673 }