sys/contrib/openzfs/module/zfs/refcount.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2012, 2021 by Delphix. All rights reserved.
  24  */
  25
  26 #include <sys/zfs_context.h>
  27 #include <sys/zfs_refcount.h>
  28
  29 #ifdef  ZFS_DEBUG
  30 /*
  31  * Reference count tracking is disabled by default.  It's memory requirements
  32  * are reasonable, however as implemented it consumes a significant amount of
  33  * cpu time.  Until its performance is improved it should be manually enabled.
  34  */
  35 int reference_tracking_enable = B_FALSE;
  36 static int reference_history = 3; /* tunable */
  37
  38 static kmem_cache_t *reference_cache;
  39 static kmem_cache_t *reference_history_cache;
  40
  41 void
  42 zfs_refcount_init(void)
  43 {
  44         reference_cache = kmem_cache_create("reference_cache",
  45             sizeof (reference_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
  46
  47         reference_history_cache = kmem_cache_create("reference_history_cache",
  48             sizeof (uint64_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
  49 }
  50
  51 void
  52 zfs_refcount_fini(void)
  53 {
  54         kmem_cache_destroy(reference_cache);
  55         kmem_cache_destroy(reference_history_cache);
  56 }
  57
  58 void
  59 zfs_refcount_create(zfs_refcount_t *rc)
  60 {
  61         mutex_init(&rc->rc_mtx, NULL, MUTEX_DEFAULT, NULL);
  62         list_create(&rc->rc_list, sizeof (reference_t),
  63             offsetof(reference_t, ref_link));
  64         list_create(&rc->rc_removed, sizeof (reference_t),
  65             offsetof(reference_t, ref_link));
  66         rc->rc_count = 0;
  67         rc->rc_removed_count = 0;
  68         rc->rc_tracked = reference_tracking_enable;
  69 }
  70
  71 void
  72 zfs_refcount_create_tracked(zfs_refcount_t *rc)
  73 {
  74         zfs_refcount_create(rc);
  75         rc->rc_tracked = B_TRUE;
  76 }
  77
  78 void
  79 zfs_refcount_create_untracked(zfs_refcount_t *rc)
  80 {
  81         zfs_refcount_create(rc);
  82         rc->rc_tracked = B_FALSE;
  83 }
  84
  85 void
  86 zfs_refcount_destroy_many(zfs_refcount_t *rc, uint64_t number)
  87 {
  88         reference_t *ref;
  89
  90         ASSERT3U(rc->rc_count, ==, number);
  91         while ((ref = list_head(&rc->rc_list))) {
  92                 list_remove(&rc->rc_list, ref);
  93                 kmem_cache_free(reference_cache, ref);
  94         }
  95         list_destroy(&rc->rc_list);
  96
  97         while ((ref = list_head(&rc->rc_removed))) {
  98                 list_remove(&rc->rc_removed, ref);
  99                 kmem_cache_free(reference_history_cache, ref->ref_removed);
 100                 kmem_cache_free(reference_cache, ref);
 101         }
 102         list_destroy(&rc->rc_removed);
 103         mutex_destroy(&rc->rc_mtx);
 104 }
 105
 106 void
 107 zfs_refcount_destroy(zfs_refcount_t *rc)
 108 {
 109         zfs_refcount_destroy_many(rc, 0);
 110 }
 111
 112 int
 113 zfs_refcount_is_zero(zfs_refcount_t *rc)
 114 {
 115         return (zfs_refcount_count(rc) == 0);
 116 }
 117
 118 int64_t
 119 zfs_refcount_count(zfs_refcount_t *rc)
 120 {
 121         return (atomic_load_64(&rc->rc_count));
 122 }
 123
 124 int64_t
 125 zfs_refcount_add_many(zfs_refcount_t *rc, uint64_t number, const void *holder)
 126 {
 127         reference_t *ref = NULL;
 128         int64_t count;
 129
 130         if (!rc->rc_tracked) {
 131                 count = atomic_add_64_nv(&(rc)->rc_count, number);
 132                 ASSERT3U(count, >=, number);
 133                 return (count);
 134         }
 135
 136         ref = kmem_cache_alloc(reference_cache, KM_SLEEP);
 137         ref->ref_holder = holder;
 138         ref->ref_number = number;
 139         mutex_enter(&rc->rc_mtx);
 140         ASSERT3U(rc->rc_count, >=, 0);
 141         list_insert_head(&rc->rc_list, ref);
 142         rc->rc_count += number;
 143         count = rc->rc_count;
 144         mutex_exit(&rc->rc_mtx);
 145
 146         return (count);
 147 }
 148
 149 int64_t
 150 zfs_refcount_add(zfs_refcount_t *rc, const void *holder)
 151 {
 152         return (zfs_refcount_add_many(rc, 1, holder));
 153 }
 154
 155 int64_t
 156 zfs_refcount_remove_many(zfs_refcount_t *rc, uint64_t number,
 157     const void *holder)
 158 {
 159         reference_t *ref;
 160         int64_t count;
 161
 162         if (!rc->rc_tracked) {
 163                 count = atomic_add_64_nv(&(rc)->rc_count, -number);
 164                 ASSERT3S(count, >=, 0);
 165                 return (count);
 166         }
 167
 168         mutex_enter(&rc->rc_mtx);
 169         ASSERT3U(rc->rc_count, >=, number);
 170         for (ref = list_head(&rc->rc_list); ref;
 171             ref = list_next(&rc->rc_list, ref)) {
 172                 if (ref->ref_holder == holder && ref->ref_number == number) {
 173                         list_remove(&rc->rc_list, ref);
 174                         if (reference_history > 0) {
 175                                 ref->ref_removed =
 176                                     kmem_cache_alloc(reference_history_cache,
 177                                     KM_SLEEP);
 178                                 list_insert_head(&rc->rc_removed, ref);
 179                                 rc->rc_removed_count++;
 180                                 if (rc->rc_removed_count > reference_history) {
 181                                         ref = list_tail(&rc->rc_removed);
 182                                         list_remove(&rc->rc_removed, ref);
 183                                         kmem_cache_free(reference_history_cache,
 184                                             ref->ref_removed);
 185                                         kmem_cache_free(reference_cache, ref);
 186                                         rc->rc_removed_count--;
 187                                 }
 188                         } else {
 189                                 kmem_cache_free(reference_cache, ref);
 190                         }
 191                         rc->rc_count -= number;
 192                         count = rc->rc_count;
 193                         mutex_exit(&rc->rc_mtx);
 194                         return (count);
 195                 }
 196         }
 197         panic("No such hold %p on refcount %llx", holder,
 198             (u_longlong_t)(uintptr_t)rc);
 199         return (-1);
 200 }
 201
 202 int64_t
 203 zfs_refcount_remove(zfs_refcount_t *rc, const void *holder)
 204 {
 205         return (zfs_refcount_remove_many(rc, 1, holder));
 206 }
 207
 208 void
 209 zfs_refcount_transfer(zfs_refcount_t *dst, zfs_refcount_t *src)
 210 {
 211         int64_t count, removed_count;
 212         list_t list, removed;
 213
 214         list_create(&list, sizeof (reference_t),
 215             offsetof(reference_t, ref_link));
 216         list_create(&removed, sizeof (reference_t),
 217             offsetof(reference_t, ref_link));
 218
 219         mutex_enter(&src->rc_mtx);
 220         count = src->rc_count;
 221         removed_count = src->rc_removed_count;
 222         src->rc_count = 0;
 223         src->rc_removed_count = 0;
 224         list_move_tail(&list, &src->rc_list);
 225         list_move_tail(&removed, &src->rc_removed);
 226         mutex_exit(&src->rc_mtx);
 227
 228         mutex_enter(&dst->rc_mtx);
 229         dst->rc_count += count;
 230         dst->rc_removed_count += removed_count;
 231         list_move_tail(&dst->rc_list, &list);
 232         list_move_tail(&dst->rc_removed, &removed);
 233         mutex_exit(&dst->rc_mtx);
 234
 235         list_destroy(&list);
 236         list_destroy(&removed);
 237 }
 238
 239 void
 240 zfs_refcount_transfer_ownership_many(zfs_refcount_t *rc, uint64_t number,
 241     const void *current_holder, const void *new_holder)
 242 {
 243         reference_t *ref;
 244         boolean_t found = B_FALSE;
 245
 246         if (!rc->rc_tracked)
 247                 return;
 248
 249         mutex_enter(&rc->rc_mtx);
 250         for (ref = list_head(&rc->rc_list); ref;
 251             ref = list_next(&rc->rc_list, ref)) {
 252                 if (ref->ref_holder == current_holder &&
 253                     ref->ref_number == number) {
 254                         ref->ref_holder = new_holder;
 255                         found = B_TRUE;
 256                         break;
 257                 }
 258         }
 259         ASSERT(found);
 260         mutex_exit(&rc->rc_mtx);
 261 }
 262
 263 void
 264 zfs_refcount_transfer_ownership(zfs_refcount_t *rc, const void *current_holder,
 265     const void *new_holder)
 266 {
 267         return (zfs_refcount_transfer_ownership_many(rc, 1, current_holder,
 268             new_holder));
 269 }
 270
 271 /*
 272  * If tracking is enabled, return true if a reference exists that matches
 273  * the "holder" tag. If tracking is disabled, then return true if a reference
 274  * might be held.
 275  */
 276 boolean_t
 277 zfs_refcount_held(zfs_refcount_t *rc, const void *holder)
 278 {
 279         reference_t *ref;
 280
 281         if (!rc->rc_tracked)
 282                 return (zfs_refcount_count(rc) > 0);
 283
 284         mutex_enter(&rc->rc_mtx);
 285         for (ref = list_head(&rc->rc_list); ref;
 286             ref = list_next(&rc->rc_list, ref)) {
 287                 if (ref->ref_holder == holder) {
 288                         mutex_exit(&rc->rc_mtx);
 289                         return (B_TRUE);
 290                 }
 291         }
 292         mutex_exit(&rc->rc_mtx);
 293         return (B_FALSE);
 294 }
 295
 296 /*
 297  * If tracking is enabled, return true if a reference does not exist that
 298  * matches the "holder" tag. If tracking is disabled, always return true
 299  * since the reference might not be held.
 300  */
 301 boolean_t
 302 zfs_refcount_not_held(zfs_refcount_t *rc, const void *holder)
 303 {
 304         reference_t *ref;
 305
 306         if (!rc->rc_tracked)
 307                 return (B_TRUE);
 308
 309         mutex_enter(&rc->rc_mtx);
 310         for (ref = list_head(&rc->rc_list); ref;
 311             ref = list_next(&rc->rc_list, ref)) {
 312                 if (ref->ref_holder == holder) {
 313                         mutex_exit(&rc->rc_mtx);
 314                         return (B_FALSE);
 315                 }
 316         }
 317         mutex_exit(&rc->rc_mtx);
 318         return (B_TRUE);
 319 }
 320
 321 EXPORT_SYMBOL(zfs_refcount_create);
 322 EXPORT_SYMBOL(zfs_refcount_destroy);
 323 EXPORT_SYMBOL(zfs_refcount_is_zero);
 324 EXPORT_SYMBOL(zfs_refcount_count);
 325 EXPORT_SYMBOL(zfs_refcount_add);
 326 EXPORT_SYMBOL(zfs_refcount_remove);
 327 EXPORT_SYMBOL(zfs_refcount_held);
 328
 329 /* BEGIN CSTYLED */
 330 ZFS_MODULE_PARAM(zfs, , reference_tracking_enable, INT, ZMOD_RW,
 331         "Track reference holders to refcount_t objects");
 332
 333 ZFS_MODULE_PARAM(zfs, , reference_history, INT, ZMOD_RW,
 334         "Maximum reference holders being tracked");
 335 /* END CSTYLED */
 336 #endif  /* ZFS_DEBUG */