sys/contrib/ck/include/ck_swlock.h

   1 /*
   2  * Copyright 2014 Jaidev Sridhar.
   3  * Copyright 2014 Samy Al Bahra.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #ifndef CK_SWLOCK_H
  29 #define CK_SWLOCK_H
  30
  31 #include <ck_elide.h>
  32 #include <ck_limits.h>
  33 #include <ck_pr.h>
  34 #include <ck_stdbool.h>
  35 #include <ck_stddef.h>
  36
  37 struct ck_swlock {
  38         uint32_t value;
  39 };
  40 typedef struct ck_swlock ck_swlock_t;
  41
  42 #define CK_SWLOCK_INITIALIZER   {0}
  43 #define CK_SWLOCK_WRITER_BIT    (1UL << 31)
  44 #define CK_SWLOCK_LATCH_BIT     (1UL << 30)
  45 #define CK_SWLOCK_WRITER_MASK   (CK_SWLOCK_LATCH_BIT | CK_SWLOCK_WRITER_BIT)
  46 #define CK_SWLOCK_READER_MASK   (UINT32_MAX ^ CK_SWLOCK_WRITER_MASK)
  47
  48 CK_CC_INLINE static void
  49 ck_swlock_init(struct ck_swlock *rw)
  50 {
  51
  52         rw->value = 0;
  53         ck_pr_barrier();
  54         return;
  55 }
  56
  57 CK_CC_INLINE static void
  58 ck_swlock_write_unlock(ck_swlock_t *rw)
  59 {
  60
  61         ck_pr_fence_unlock();
  62         ck_pr_and_32(&rw->value, CK_SWLOCK_READER_MASK);
  63         return;
  64 }
  65
  66 CK_CC_INLINE static bool
  67 ck_swlock_locked_writer(ck_swlock_t *rw)
  68 {
  69         bool r;
  70
  71         r = ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT;
  72         ck_pr_fence_acquire();
  73         return r;
  74 }
  75
  76 CK_CC_INLINE static void
  77 ck_swlock_write_downgrade(ck_swlock_t *rw)
  78 {
  79
  80         ck_pr_inc_32(&rw->value);
  81         ck_swlock_write_unlock(rw);
  82         return;
  83 }
  84
  85 CK_CC_INLINE static bool
  86 ck_swlock_locked(ck_swlock_t *rw)
  87 {
  88         bool r;
  89
  90         r = ck_pr_load_32(&rw->value);
  91         ck_pr_fence_acquire();
  92         return r;
  93 }
  94
  95 CK_CC_INLINE static bool
  96 ck_swlock_write_trylock(ck_swlock_t *rw)
  97 {
  98         bool r;
  99
 100         r = ck_pr_cas_32(&rw->value, 0, CK_SWLOCK_WRITER_BIT);
 101         ck_pr_fence_lock();
 102         return r;
 103 }
 104
 105 CK_ELIDE_TRYLOCK_PROTOTYPE(ck_swlock_write, ck_swlock_t,
 106     ck_swlock_locked, ck_swlock_write_trylock)
 107
 108 CK_CC_INLINE static void
 109 ck_swlock_write_lock(ck_swlock_t *rw)
 110 {
 111
 112         ck_pr_or_32(&rw->value, CK_SWLOCK_WRITER_BIT);
 113         while (ck_pr_load_32(&rw->value) & CK_SWLOCK_READER_MASK)
 114                 ck_pr_stall();
 115
 116         ck_pr_fence_lock();
 117         return;
 118 }
 119
 120 CK_CC_INLINE static void
 121 ck_swlock_write_latch(ck_swlock_t *rw)
 122 {
 123
 124         /* Publish intent to acquire lock. */
 125         ck_pr_or_32(&rw->value, CK_SWLOCK_WRITER_BIT);
 126
 127         /* Stall until readers have seen the writer and cleared. */
 128         while (ck_pr_cas_32(&rw->value, CK_SWLOCK_WRITER_BIT,
 129             CK_SWLOCK_WRITER_MASK) == false)  {
 130                 do {
 131                         ck_pr_stall();
 132                 } while (ck_pr_load_32(&rw->value) != CK_SWLOCK_WRITER_BIT);
 133         }
 134
 135         ck_pr_fence_lock();
 136         return;
 137 }
 138
 139 CK_CC_INLINE static void
 140 ck_swlock_write_unlatch(ck_swlock_t *rw)
 141 {
 142
 143         ck_pr_fence_unlock();
 144         ck_pr_store_32(&rw->value, 0);
 145         return;
 146 }
 147
 148 CK_ELIDE_PROTOTYPE(ck_swlock_write, ck_swlock_t,
 149     ck_swlock_locked, ck_swlock_write_lock,
 150     ck_swlock_locked_writer, ck_swlock_write_unlock)
 151
 152 CK_ELIDE_TRYLOCK_PROTOTYPE(ck_swlock_read, ck_swlock_t,
 153     ck_swlock_locked_writer, ck_swlock_read_trylock)
 154
 155 CK_CC_INLINE static bool
 156 ck_swlock_read_trylock(ck_swlock_t *rw)
 157 {
 158         uint32_t l = ck_pr_load_32(&rw->value);
 159
 160         if (l & CK_SWLOCK_WRITER_BIT)
 161                 return false;
 162
 163         l = ck_pr_faa_32(&rw->value, 1) & CK_SWLOCK_WRITER_MASK;
 164         if (l == CK_SWLOCK_WRITER_BIT)
 165                 ck_pr_dec_32(&rw->value);
 166
 167         ck_pr_fence_lock();
 168         return l == 0;
 169 }
 170
 171 CK_CC_INLINE static void
 172 ck_swlock_read_lock(ck_swlock_t *rw)
 173 {
 174         uint32_t l;
 175
 176         for (;;) {
 177                 while (ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT)
 178                         ck_pr_stall();
 179
 180                 l = ck_pr_faa_32(&rw->value, 1) & CK_SWLOCK_WRITER_MASK;
 181                 if (l == 0)
 182                         break;
 183
 184                 /*
 185                  * If the latch bit has not been set, then the writer would
 186                  * have observed the reader and will wait to completion of
 187                  * read-side critical section.
 188                  */
 189                 if (l == CK_SWLOCK_WRITER_BIT)
 190                         ck_pr_dec_32(&rw->value);
 191         }
 192
 193         ck_pr_fence_lock();
 194         return;
 195 }
 196
 197 CK_CC_INLINE static bool
 198 ck_swlock_locked_reader(ck_swlock_t *rw)
 199 {
 200
 201         ck_pr_fence_load();
 202         return ck_pr_load_32(&rw->value) & CK_SWLOCK_READER_MASK;
 203 }
 204
 205 CK_CC_INLINE static void
 206 ck_swlock_read_unlock(ck_swlock_t *rw)
 207 {
 208
 209         ck_pr_fence_unlock();
 210         ck_pr_dec_32(&rw->value);
 211         return;
 212 }
 213
 214 CK_ELIDE_PROTOTYPE(ck_swlock_read, ck_swlock_t,
 215     ck_swlock_locked_writer, ck_swlock_read_lock,
 216     ck_swlock_locked_reader, ck_swlock_read_unlock)
 217
 218 #endif /* CK_SWLOCK_H */