lib/libthr/thread/thr_pshared.c

   1 /*-
   2  * Copyright (c) 2015 The FreeBSD Foundation
   3  *
   4  * This software was developed by Konstantin Belousov
   5  * under sponsorship from the FreeBSD Foundation.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 #include <sys/types.h>
  30 #include <sys/mman.h>
  31 #include <sys/queue.h>
  32 #include "namespace.h"
  33 #include <stdlib.h>
  34 #include "un-namespace.h"
  35
  36 #include "thr_private.h"
  37
  38 struct psh {
  39         LIST_ENTRY(psh) link;
  40         void *key;
  41         void *val;
  42 };
  43
  44 LIST_HEAD(pshared_hash_head, psh);
  45 #define HASH_SIZE       128
  46 static struct pshared_hash_head pshared_hash[HASH_SIZE];
  47 #define PSHARED_KEY_HASH(key)   (((unsigned long)(key) >> 8) % HASH_SIZE)
  48 /* XXXKIB: lock could be split to per-hash chain, if appears contested */
  49 static struct urwlock pshared_lock = DEFAULT_URWLOCK;
  50 static int page_size;
  51
  52 void
  53 __thr_pshared_init(void)
  54 {
  55         int i;
  56
  57         page_size = getpagesize();
  58         THR_ASSERT(page_size >= THR_PAGE_SIZE_MIN,
  59             "THR_PAGE_SIZE_MIN is too large");
  60
  61         _thr_urwlock_init(&pshared_lock);
  62         for (i = 0; i < HASH_SIZE; i++)
  63                 LIST_INIT(&pshared_hash[i]);
  64 }
  65
  66 static void
  67 pshared_rlock(struct pthread *curthread)
  68 {
  69
  70         curthread->locklevel++;
  71         _thr_rwl_rdlock(&pshared_lock);
  72 }
  73
  74 static void
  75 pshared_wlock(struct pthread *curthread)
  76 {
  77
  78         curthread->locklevel++;
  79         _thr_rwl_wrlock(&pshared_lock);
  80 }
  81
  82 static void
  83 pshared_unlock(struct pthread *curthread)
  84 {
  85
  86         _thr_rwl_unlock(&pshared_lock);
  87         curthread->locklevel--;
  88         _thr_ast(curthread);
  89 }
  90
  91 /*
  92  * Among all processes sharing a lock only one executes
  93  * pthread_lock_destroy().  Other processes still have the hash and
  94  * mapped off-page.
  95  *
  96  * Mitigate the problem by checking the liveness of all hashed keys
  97  * periodically.  Right now this is executed on each
  98  * pthread_lock_destroy(), but may be done less often if found to be
  99  * too time-consuming.
 100  */
 101 static void
 102 pshared_gc(struct pthread *curthread)
 103 {
 104         struct pshared_hash_head *hd;
 105         struct psh *h, *h1;
 106         int error, i;
 107
 108         pshared_wlock(curthread);
 109         for (i = 0; i < HASH_SIZE; i++) {
 110                 hd = &pshared_hash[i];
 111                 LIST_FOREACH_SAFE(h, hd, link, h1) {
 112                         error = _umtx_op(NULL, UMTX_OP_SHM, UMTX_SHM_ALIVE,
 113                             h->val, NULL);
 114                         if (error == 0)
 115                                 continue;
 116                         LIST_REMOVE(h, link);
 117                         munmap(h->val, page_size);
 118                         free(h);
 119                 }
 120         }
 121         pshared_unlock(curthread);
 122 }
 123
 124 static void *
 125 pshared_lookup(void *key)
 126 {
 127         struct pshared_hash_head *hd;
 128         struct psh *h;
 129
 130         hd = &pshared_hash[PSHARED_KEY_HASH(key)];
 131         LIST_FOREACH(h, hd, link) {
 132                 if (h->key == key)
 133                         return (h->val);
 134         }
 135         return (NULL);
 136 }
 137
 138 static int
 139 pshared_insert(void *key, void **val)
 140 {
 141         struct pshared_hash_head *hd;
 142         struct psh *h;
 143
 144         hd = &pshared_hash[PSHARED_KEY_HASH(key)];
 145         LIST_FOREACH(h, hd, link) {
 146                 /*
 147                  * When the key already exists in the hash, we should
 148                  * return either the new (just mapped) or old (hashed)
 149                  * val, and the other val should be unmapped to avoid
 150                  * address space leak.
 151                  *
 152                  * If two threads perform lock of the same object
 153                  * which is not yet stored in the pshared_hash, then
 154                  * the val already inserted by the first thread should
 155                  * be returned, and the second val freed (order is by
 156                  * the pshared_lock()).  Otherwise, if we unmap the
 157                  * value obtained from the hash, the first thread
 158                  * might operate on an unmapped off-page object.
 159                  *
 160                  * There is still an issue: if hashed key was unmapped
 161                  * and then other page is mapped at the same key
 162                  * address, the hash would return the old val.  I
 163                  * decided to handle the race of simultaneous hash
 164                  * insertion, leaving the unlikely remap problem
 165                  * unaddressed.
 166                  */
 167                 if (h->key == key) {
 168                         if (h->val != *val) {
 169                                 munmap(*val, page_size);
 170                                 *val = h->val;
 171                         }
 172                         return (1);
 173                 }
 174         }
 175
 176         h = malloc(sizeof(*h));
 177         if (h == NULL)
 178                 return (0);
 179         h->key = key;
 180         h->val = *val;
 181         LIST_INSERT_HEAD(hd, h, link);
 182         return (1);
 183 }
 184
 185 static void *
 186 pshared_remove(void *key)
 187 {
 188         struct pshared_hash_head *hd;
 189         struct psh *h;
 190         void *val;
 191
 192         hd = &pshared_hash[PSHARED_KEY_HASH(key)];
 193         LIST_FOREACH(h, hd, link) {
 194                 if (h->key == key) {
 195                         LIST_REMOVE(h, link);
 196                         val = h->val;
 197                         free(h);
 198                         return (val);
 199                 }
 200         }
 201         return (NULL);
 202 }
 203
 204 static void
 205 pshared_clean(void *key, void *val)
 206 {
 207
 208         if (val != NULL)
 209                 munmap(val, page_size);
 210         _umtx_op(NULL, UMTX_OP_SHM, UMTX_SHM_DESTROY, key, NULL);
 211 }
 212
 213 static void
 214 pshared_destroy(struct pthread *curthread, void *key)
 215 {
 216         void *val;
 217
 218         pshared_wlock(curthread);
 219         val = pshared_remove(key);
 220         pshared_unlock(curthread);
 221         pshared_clean(key, val);
 222 }
 223
 224 void *
 225 __thr_pshared_offpage(void *key, int doalloc)
 226 {
 227         struct pthread *curthread;
 228         void *res;
 229         int fd, ins_done;
 230
 231         curthread = _get_curthread();
 232         if (doalloc) {
 233                 pshared_destroy(curthread, key);
 234                 res = NULL;
 235         } else {
 236                 pshared_rlock(curthread);
 237                 res = pshared_lookup(key);
 238                 pshared_unlock(curthread);
 239                 if (res != NULL)
 240                         return (res);
 241         }
 242         fd = _umtx_op(NULL, UMTX_OP_SHM, doalloc ? UMTX_SHM_CREAT :
 243             UMTX_SHM_LOOKUP, key, NULL);
 244         if (fd == -1)
 245                 return (NULL);
 246         res = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
 247         close(fd);
 248         if (res == MAP_FAILED)
 249                 return (NULL);
 250         pshared_wlock(curthread);
 251         ins_done = pshared_insert(key, &res);
 252         pshared_unlock(curthread);
 253         if (!ins_done) {
 254                 pshared_clean(key, res);
 255                 res = NULL;
 256         }
 257         return (res);
 258 }
 259
 260 void
 261 __thr_pshared_destroy(void *key)
 262 {
 263         struct pthread *curthread;
 264
 265         curthread = _get_curthread();
 266         pshared_destroy(curthread, key);
 267         pshared_gc(curthread);
 268 }
 269
 270 void
 271 __thr_pshared_atfork_pre(void)
 272 {
 273
 274         _thr_rwl_rdlock(&pshared_lock);
 275 }
 276
 277 void
 278 __thr_pshared_atfork_post(void)
 279 {
 280
 281         _thr_rwl_unlock(&pshared_lock);
 282 }