lib/libthr/thread/thr_umtx.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2005 David Xu <davidxu@freebsd.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  *
  28  * $FreeBSD$
  29  */
  30
  31 #ifndef _THR_FBSD_UMTX_H_
  32 #define _THR_FBSD_UMTX_H_
  33
  34 #include <strings.h>
  35 #include <sys/umtx.h>
  36
  37 #ifdef __LP64__
  38 #define DEFAULT_UMUTEX  {0,0,{0,0},0,{0,0}}
  39 #else
  40 #define DEFAULT_UMUTEX  {0,0,{0,0},0,0,{0,0}}
  41 #endif
  42 #define DEFAULT_URWLOCK {0,0,0,0,{0,0,0,0}}
  43
  44 int _umtx_op_err(void *, int op, u_long, void *, void *) __hidden;
  45 int __thr_umutex_lock(struct umutex *mtx, uint32_t id) __hidden;
  46 int __thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) __hidden;
  47 int __thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
  48         const struct timespec *timeout) __hidden;
  49 int __thr_umutex_unlock(struct umutex *mtx) __hidden;
  50 int __thr_umutex_trylock(struct umutex *mtx) __hidden;
  51 int __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling,
  52         uint32_t *oldceiling) __hidden;
  53
  54 void _thr_umutex_init(struct umutex *mtx) __hidden;
  55 void _thr_urwlock_init(struct urwlock *rwl) __hidden;
  56
  57 int _thr_umtx_wait(volatile long *mtx, long exp,
  58         const struct timespec *timeout) __hidden;
  59 int _thr_umtx_wait_uint(volatile u_int *mtx, u_int exp,
  60         const struct timespec *timeout, int shared) __hidden;
  61 int _thr_umtx_timedwait_uint(volatile u_int *mtx, u_int exp, int clockid,
  62         const struct timespec *timeout, int shared) __hidden;
  63 int _thr_umtx_wake(volatile void *mtx, int count, int shared) __hidden;
  64 int _thr_ucond_wait(struct ucond *cv, struct umutex *m,
  65         const struct timespec *timeout, int flags) __hidden;
  66 void _thr_ucond_init(struct ucond *cv) __hidden;
  67 int _thr_ucond_signal(struct ucond *cv) __hidden;
  68 int _thr_ucond_broadcast(struct ucond *cv) __hidden;
  69
  70 int __thr_rwlock_rdlock(struct urwlock *rwlock, int flags,
  71         const struct timespec *tsp) __hidden;
  72 int __thr_rwlock_wrlock(struct urwlock *rwlock,
  73         const struct timespec *tsp) __hidden;
  74 int __thr_rwlock_unlock(struct urwlock *rwlock) __hidden;
  75
  76 /* Internal used only */
  77 void _thr_rwl_rdlock(struct urwlock *rwlock) __hidden;
  78 void _thr_rwl_wrlock(struct urwlock *rwlock) __hidden;
  79 void _thr_rwl_unlock(struct urwlock *rwlock) __hidden;
  80
  81 static inline int
  82 _thr_umutex_trylock(struct umutex *mtx, uint32_t id)
  83 {
  84
  85         if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id))
  86                 return (0);
  87         if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_OWNERDEAD) &&
  88             atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_RB_OWNERDEAD,
  89             id | UMUTEX_CONTESTED))
  90                 return (EOWNERDEAD);
  91         if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_NOTRECOV))
  92                 return (ENOTRECOVERABLE);
  93         if ((mtx->m_flags & UMUTEX_PRIO_PROTECT) == 0)
  94                 return (EBUSY);
  95         return (__thr_umutex_trylock(mtx));
  96 }
  97
  98 static inline int
  99 _thr_umutex_trylock2(struct umutex *mtx, uint32_t id)
 100 {
 101
 102         if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id) != 0)
 103                 return (0);
 104         if ((uint32_t)mtx->m_owner == UMUTEX_CONTESTED &&
 105             __predict_true((mtx->m_flags & (UMUTEX_PRIO_PROTECT |
 106            UMUTEX_PRIO_INHERIT)) == 0) &&
 107            atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_CONTESTED,
 108            id | UMUTEX_CONTESTED))
 109                 return (0);
 110         if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_OWNERDEAD) &&
 111             atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_RB_OWNERDEAD,
 112             id | UMUTEX_CONTESTED))
 113                 return (EOWNERDEAD);
 114         if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_NOTRECOV))
 115                 return (ENOTRECOVERABLE);
 116         return (EBUSY);
 117 }
 118
 119 static inline int
 120 _thr_umutex_lock(struct umutex *mtx, uint32_t id)
 121 {
 122
 123         if (_thr_umutex_trylock2(mtx, id) == 0)
 124                 return (0);
 125         return (__thr_umutex_lock(mtx, id));
 126 }
 127
 128 static inline int
 129 _thr_umutex_lock_spin(struct umutex *mtx, uint32_t id)
 130 {
 131
 132         if (_thr_umutex_trylock2(mtx, id) == 0)
 133                 return (0);
 134         return (__thr_umutex_lock_spin(mtx, id));
 135 }
 136
 137 static inline int
 138 _thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
 139     const struct timespec *timeout)
 140 {
 141
 142         if (_thr_umutex_trylock2(mtx, id) == 0)
 143                 return (0);
 144         return (__thr_umutex_timedlock(mtx, id, timeout));
 145 }
 146
 147 static inline int
 148 _thr_umutex_unlock2(struct umutex *mtx, uint32_t id, int *defer)
 149 {
 150         uint32_t flags, owner;
 151         bool noncst;
 152
 153         flags = mtx->m_flags;
 154         noncst = (flags & UMUTEX_NONCONSISTENT) != 0;
 155
 156         if ((flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0) {
 157                 if (atomic_cmpset_rel_32(&mtx->m_owner, id, noncst ?
 158                     UMUTEX_RB_NOTRECOV : UMUTEX_UNOWNED))
 159                         return (0);
 160                 return (__thr_umutex_unlock(mtx));
 161         }
 162
 163         do {
 164                 owner = mtx->m_owner;
 165                 if (__predict_false((owner & ~UMUTEX_CONTESTED) != id))
 166                         return (EPERM);
 167         } while (__predict_false(!atomic_cmpset_rel_32(&mtx->m_owner, owner,
 168             noncst ? UMUTEX_RB_NOTRECOV : UMUTEX_UNOWNED)));
 169         if ((owner & UMUTEX_CONTESTED) != 0) {
 170                 if (defer == NULL || noncst)
 171                         (void)_umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE2,
 172                             flags, 0, 0);
 173                 else
 174                         *defer = 1;
 175         }
 176         return (0);
 177 }
 178
 179 static inline int
 180 _thr_umutex_unlock(struct umutex *mtx, uint32_t id)
 181 {
 182
 183         return (_thr_umutex_unlock2(mtx, id, NULL));
 184 }
 185
 186 static inline int
 187 _thr_rwlock_tryrdlock(struct urwlock *rwlock, int flags)
 188 {
 189         int32_t state, wrflags;
 190
 191         if ((flags & URWLOCK_PREFER_READER) != 0 ||
 192             (rwlock->rw_flags & URWLOCK_PREFER_READER) != 0)
 193                 wrflags = URWLOCK_WRITE_OWNER;
 194         else
 195                 wrflags = URWLOCK_WRITE_OWNER | URWLOCK_WRITE_WAITERS;
 196         state = rwlock->rw_state;
 197         while (!(state & wrflags)) {
 198                 if (__predict_false(URWLOCK_READER_COUNT(state) ==
 199                     URWLOCK_MAX_READERS))
 200                         return (EAGAIN);
 201                 if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state + 1))
 202                         return (0);
 203                 state = rwlock->rw_state;
 204         }
 205
 206         return (EBUSY);
 207 }
 208
 209 static inline int
 210 _thr_rwlock_trywrlock(struct urwlock *rwlock)
 211 {
 212         int32_t state;
 213
 214         state = rwlock->rw_state;
 215         while ((state & URWLOCK_WRITE_OWNER) == 0 &&
 216             URWLOCK_READER_COUNT(state) == 0) {
 217                 if (atomic_cmpset_acq_32(&rwlock->rw_state, state,
 218                     state | URWLOCK_WRITE_OWNER))
 219                         return (0);
 220                 state = rwlock->rw_state;
 221         }
 222
 223         return (EBUSY);
 224 }
 225
 226 static inline int
 227 _thr_rwlock_rdlock(struct urwlock *rwlock, int flags, struct timespec *tsp)
 228 {
 229
 230         if (_thr_rwlock_tryrdlock(rwlock, flags) == 0)
 231                 return (0);
 232         return (__thr_rwlock_rdlock(rwlock, flags, tsp));
 233 }
 234
 235 static inline int
 236 _thr_rwlock_wrlock(struct urwlock *rwlock, struct timespec *tsp)
 237 {
 238
 239         if (_thr_rwlock_trywrlock(rwlock) == 0)
 240                 return (0);
 241         return (__thr_rwlock_wrlock(rwlock, tsp));
 242 }
 243
 244 static inline int
 245 _thr_rwlock_unlock(struct urwlock *rwlock)
 246 {
 247         int32_t state;
 248
 249         state = rwlock->rw_state;
 250         if ((state & URWLOCK_WRITE_OWNER) != 0) {
 251                 if (atomic_cmpset_rel_32(&rwlock->rw_state,
 252                     URWLOCK_WRITE_OWNER, 0))
 253                         return (0);
 254         } else {
 255                 for (;;) {
 256                         if (__predict_false(URWLOCK_READER_COUNT(state) == 0))
 257                                 return (EPERM);
 258                         if (!((state & (URWLOCK_WRITE_WAITERS |
 259                             URWLOCK_READ_WAITERS)) != 0 &&
 260                             URWLOCK_READER_COUNT(state) == 1)) {
 261                                 if (atomic_cmpset_rel_32(&rwlock->rw_state,
 262                                     state, state - 1))
 263                                         return (0);
 264                                 state = rwlock->rw_state;
 265                         } else {
 266                                 break;
 267                         }
 268                 }
 269         }
 270         return (__thr_rwlock_unlock(rwlock));
 271 }
 272 #endif