1 /* caching.c : in-memory caching
3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
20 * ====================================================================
28 #include "temp_serializer.h"
29 #include "../libsvn_fs/fs-loader.h"
31 #include "svn_config.h"
32 #include "svn_cache_config.h"
34 #include "svn_private_config.h"
36 #include "svn_pools.h"
38 #include "private/svn_debug.h"
39 #include "private/svn_subr_private.h"
41 /* Take the ORIGINAL string and replace all occurrences of ":" without
42 * limiting the key space. Allocate the result in POOL.
45 normalize_key_part(const char *original,
49 apr_size_t len = strlen(original);
50 svn_stringbuf_t *normalized = svn_stringbuf_create_ensure(len, pool);
52 for (i = 0; i < len; ++i)
57 case ':': svn_stringbuf_appendbytes(normalized, "%_", 2);
59 case '%': svn_stringbuf_appendbytes(normalized, "%%", 2);
61 default : svn_stringbuf_appendbyte(normalized, c);
65 return normalized->data;
68 /* Return a memcache in *MEMCACHE_P for FS if it's configured to use
69 memcached, or NULL otherwise. Also, sets *FAIL_STOP to a boolean
70 indicating whether cache errors should be returned to the caller or
71 just passed to the FS warning handler.
73 *CACHE_TXDELTAS, *CACHE_FULLTEXTS and *CACHE_REVPROPS flags will be set
74 according to FS->CONFIG. *CACHE_NAMESPACE receives the cache prefix
77 Use FS->pool for allocating the memcache and CACHE_NAMESPACE, and POOL
78 for temporary allocations. */
80 read_config(svn_memcache_t **memcache_p,
81 svn_boolean_t *fail_stop,
82 const char **cache_namespace,
83 svn_boolean_t *cache_txdeltas,
84 svn_boolean_t *cache_fulltexts,
85 svn_boolean_t *cache_revprops,
89 fs_fs_data_t *ffd = fs->fsap_data;
91 SVN_ERR(svn_cache__make_memcache_from_config(memcache_p, ffd->config,
94 /* No cache namespace by default. I.e. all FS instances share the
95 * cached data. If you specify different namespaces, the data will
96 * share / compete for the same cache memory but keys will not match
97 * across namespaces and, thus, cached data will not be shared between
100 * Since the namespace will be concatenated with other elements to form
101 * the complete key prefix, we must make sure that the resulting string
102 * is unique and cannot be created by any other combination of elements.
105 = normalize_key_part(svn_hash__get_cstring(fs->config,
106 SVN_FS_CONFIG_FSFS_CACHE_NS,
110 /* don't cache text deltas by default.
111 * Once we reconstructed the fulltexts from the deltas,
112 * these deltas are rarely re-used. Therefore, only tools
113 * like svnadmin will activate this to speed up operations
117 = svn_hash__get_bool(fs->config,
118 SVN_FS_CONFIG_FSFS_CACHE_DELTAS,
120 /* by default, cache fulltexts.
121 * Most SVN tools care about reconstructed file content.
122 * Thus, this is a reasonable default.
123 * SVN admin tools may set that to FALSE because fulltexts
124 * won't be re-used rendering the cache less effective
125 * by squeezing wanted data out.
128 = svn_hash__get_bool(fs->config,
129 SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS,
132 /* don't cache revprops by default.
133 * Revprop caching significantly speeds up operations like
134 * svn ls -v. However, it requires synchronization that may
135 * not be available or efficient in the current server setup.
137 * If the caller chose option "2", enable revprop caching if
138 * the required API support is there to make it efficient.
140 if (strcmp(svn_hash__get_cstring(fs->config,
141 SVN_FS_CONFIG_FSFS_CACHE_REVPROPS,
144 = svn_hash__get_bool(fs->config,
145 SVN_FS_CONFIG_FSFS_CACHE_REVPROPS,
148 *cache_revprops = svn_named_atomic__is_efficient();
150 return svn_config_get_bool(ffd->config, fail_stop,
151 CONFIG_SECTION_CACHES, CONFIG_OPTION_FAIL_STOP,
156 /* Implements svn_cache__error_handler_t
157 * This variant clears the error after logging it.
160 warn_and_continue_on_cache_errors(svn_error_t *err,
164 svn_fs_t *fs = baton;
165 (fs->warning)(fs->warning_baton, err);
166 svn_error_clear(err);
171 /* Implements svn_cache__error_handler_t
172 * This variant logs the error and passes it on to the callers.
175 warn_and_fail_on_cache_errors(svn_error_t *err,
179 svn_fs_t *fs = baton;
180 (fs->warning)(fs->warning_baton, err);
184 #ifdef SVN_DEBUG_CACHE_DUMP_STATS
185 /* Baton to be used for the dump_cache_statistics() pool cleanup function, */
186 struct dump_cache_baton_t
188 /* the pool about to be cleaned up. Will be used for temp. allocations. */
191 /* the cache to dump the statistics for */
195 /* APR pool cleanup handler that will printf the statistics of the
196 cache referenced by the baton in BATON_VOID. */
198 dump_cache_statistics(void *baton_void)
200 struct dump_cache_baton_t *baton = baton_void;
202 apr_status_t result = APR_SUCCESS;
203 svn_cache__info_t info;
204 svn_string_t *text_stats;
205 apr_array_header_t *lines;
208 svn_error_t *err = svn_cache__get_info(baton->cache,
215 text_stats = svn_cache__format_info(&info, baton->pool);
216 lines = svn_cstring_split(text_stats->data, "\n", FALSE, baton->pool);
218 for (i = 0; i < lines->nelts; ++i)
220 const char *line = APR_ARRAY_IDX(lines, i, const char *);
222 SVN_DBG(("%s\n", line));
227 /* process error returns */
230 result = err->apr_err;
231 svn_error_clear(err);
236 #endif /* SVN_DEBUG_CACHE_DUMP_STATS */
238 /* This function sets / registers the required callbacks for a given
239 * not transaction-specific CACHE object in FS, if CACHE is not NULL.
241 * All these svn_cache__t instances shall be handled uniformly. Unless
242 * ERROR_HANDLER is NULL, register it for the given CACHE in FS.
245 init_callbacks(svn_cache__t *cache,
247 svn_cache__error_handler_t error_handler,
252 #ifdef SVN_DEBUG_CACHE_DUMP_STATS
254 /* schedule printing the access statistics upon pool cleanup,
255 * i.e. end of FSFS session.
257 struct dump_cache_baton_t *baton;
259 baton = apr_palloc(pool, sizeof(*baton));
261 baton->cache = cache;
263 apr_pool_cleanup_register(pool,
265 dump_cache_statistics,
266 apr_pool_cleanup_null);
270 SVN_ERR(svn_cache__set_error_handler(cache,
280 /* Sets *CACHE_P to cache instance based on provided options.
281 * Creates memcache if MEMCACHE is not NULL. Creates membuffer cache if
282 * MEMBUFFER is not NULL. Fallbacks to inprocess cache if MEMCACHE and
283 * MEMBUFFER are NULL and pages is non-zero. Sets *CACHE_P to NULL
286 * Unless NO_HANDLER is true, register an error handler that reports errors
287 * as warnings to the FS warning callback.
289 * Cache is allocated in POOL.
292 create_cache(svn_cache__t **cache_p,
293 svn_memcache_t *memcache,
294 svn_membuffer_t *membuffer,
296 apr_int64_t items_per_page,
297 svn_cache__serialize_func_t serializer,
298 svn_cache__deserialize_func_t deserializer,
302 svn_boolean_t no_handler,
305 svn_cache__error_handler_t error_handler = no_handler
307 : warn_and_fail_on_cache_errors;
311 SVN_ERR(svn_cache__create_memcache(cache_p, memcache,
312 serializer, deserializer, klen,
314 error_handler = no_handler
316 : warn_and_continue_on_cache_errors;
320 SVN_ERR(svn_cache__create_membuffer_cache(
321 cache_p, membuffer, serializer, deserializer,
322 klen, prefix, FALSE, pool));
326 SVN_ERR(svn_cache__create_inprocess(
327 cache_p, serializer, deserializer, klen, pages,
328 items_per_page, FALSE, prefix, pool));
335 SVN_ERR(init_callbacks(*cache_p, fs, error_handler, pool));
341 svn_fs_fs__initialize_caches(svn_fs_t *fs,
344 fs_fs_data_t *ffd = fs->fsap_data;
345 const char *prefix = apr_pstrcat(pool,
347 "/", normalize_key_part(fs->path, pool),
350 svn_memcache_t *memcache;
351 svn_membuffer_t *membuffer;
352 svn_boolean_t no_handler;
353 svn_boolean_t cache_txdeltas;
354 svn_boolean_t cache_fulltexts;
355 svn_boolean_t cache_revprops;
356 const char *cache_namespace;
358 /* Evaluating the cache configuration. */
359 SVN_ERR(read_config(&memcache,
368 prefix = apr_pstrcat(pool, "ns:", cache_namespace, ":", prefix, NULL);
370 membuffer = svn_cache__get_global_membuffer_cache();
372 /* Make the cache for revision roots. For the vast majority of
373 * commands, this is only going to contain a few entries (svnadmin
374 * dump/verify is an exception here), so to reduce overhead let's
375 * try to keep it to just one page. I estimate each entry has about
376 * 72 bytes of overhead (svn_revnum_t key, svn_fs_id_t +
377 * id_private_t + 3 strings for value, and the cache_entry); the
378 * default pool size is 8192, so about a hundred should fit
380 SVN_ERR(create_cache(&(ffd->rev_root_id_cache),
384 svn_fs_fs__serialize_id,
385 svn_fs_fs__deserialize_id,
386 sizeof(svn_revnum_t),
387 apr_pstrcat(pool, prefix, "RRI", (char *)NULL),
392 /* Rough estimate: revision DAG nodes have size around 320 bytes, so
393 * let's put 16 on a page. */
394 SVN_ERR(create_cache(&(ffd->rev_node_cache),
398 svn_fs_fs__dag_serialize,
399 svn_fs_fs__dag_deserialize,
401 apr_pstrcat(pool, prefix, "DAG", (char *)NULL),
406 /* 1st level DAG node cache */
407 ffd->dag_node_cache = svn_fs_fs__create_dag_cache(pool);
409 /* Very rough estimate: 1K per directory. */
410 SVN_ERR(create_cache(&(ffd->dir_cache),
414 svn_fs_fs__serialize_dir_entries,
415 svn_fs_fs__deserialize_dir_entries,
417 apr_pstrcat(pool, prefix, "DIR", (char *)NULL),
422 /* Only 16 bytes per entry (a revision number + the corresponding offset).
423 Since we want ~8k pages, that means 512 entries per page. */
424 SVN_ERR(create_cache(&(ffd->packed_offset_cache),
428 svn_fs_fs__serialize_manifest,
429 svn_fs_fs__deserialize_manifest,
430 sizeof(svn_revnum_t),
431 apr_pstrcat(pool, prefix, "PACK-MANIFEST",
437 /* initialize node revision cache, if caching has been enabled */
438 SVN_ERR(create_cache(&(ffd->node_revision_cache),
441 0, 0, /* Do not use inprocess cache */
442 svn_fs_fs__serialize_node_revision,
443 svn_fs_fs__deserialize_node_revision,
444 sizeof(pair_cache_key_t),
445 apr_pstrcat(pool, prefix, "NODEREVS", (char *)NULL),
450 /* initialize node change list cache, if caching has been enabled */
451 SVN_ERR(create_cache(&(ffd->changes_cache),
454 0, 0, /* Do not use inprocess cache */
455 svn_fs_fs__serialize_changes,
456 svn_fs_fs__deserialize_changes,
457 sizeof(svn_revnum_t),
458 apr_pstrcat(pool, prefix, "CHANGES", (char *)NULL),
463 /* if enabled, cache fulltext and other derived information */
466 SVN_ERR(create_cache(&(ffd->fulltext_cache),
469 0, 0, /* Do not use inprocess cache */
470 /* Values are svn_stringbuf_t */
472 sizeof(pair_cache_key_t),
473 apr_pstrcat(pool, prefix, "TEXT", (char *)NULL),
478 SVN_ERR(create_cache(&(ffd->properties_cache),
481 0, 0, /* Do not use inprocess cache */
482 svn_fs_fs__serialize_properties,
483 svn_fs_fs__deserialize_properties,
484 sizeof(pair_cache_key_t),
485 apr_pstrcat(pool, prefix, "PROP",
491 SVN_ERR(create_cache(&(ffd->mergeinfo_cache),
494 0, 0, /* Do not use inprocess cache */
495 svn_fs_fs__serialize_mergeinfo,
496 svn_fs_fs__deserialize_mergeinfo,
498 apr_pstrcat(pool, prefix, "MERGEINFO",
504 SVN_ERR(create_cache(&(ffd->mergeinfo_existence_cache),
507 0, 0, /* Do not use inprocess cache */
508 /* Values are svn_stringbuf_t */
511 apr_pstrcat(pool, prefix, "HAS_MERGEINFO",
519 ffd->fulltext_cache = NULL;
520 ffd->properties_cache = NULL;
521 ffd->mergeinfo_cache = NULL;
522 ffd->mergeinfo_existence_cache = NULL;
525 /* initialize revprop cache, if full-text caching has been enabled */
528 SVN_ERR(create_cache(&(ffd->revprop_cache),
531 0, 0, /* Do not use inprocess cache */
532 svn_fs_fs__serialize_properties,
533 svn_fs_fs__deserialize_properties,
534 sizeof(pair_cache_key_t),
535 apr_pstrcat(pool, prefix, "REVPROP",
543 ffd->revprop_cache = NULL;
546 /* if enabled, cache text deltas and their combinations */
549 SVN_ERR(create_cache(&(ffd->txdelta_window_cache),
552 0, 0, /* Do not use inprocess cache */
553 svn_fs_fs__serialize_txdelta_window,
554 svn_fs_fs__deserialize_txdelta_window,
556 apr_pstrcat(pool, prefix, "TXDELTA_WINDOW",
562 SVN_ERR(create_cache(&(ffd->combined_window_cache),
565 0, 0, /* Do not use inprocess cache */
566 /* Values are svn_stringbuf_t */
569 apr_pstrcat(pool, prefix, "COMBINED_WINDOW",
577 ffd->txdelta_window_cache = NULL;
578 ffd->combined_window_cache = NULL;
584 /* Baton to be used for the remove_txn_cache() pool cleanup function, */
585 struct txn_cleanup_baton_t
587 /* the cache to reset */
588 svn_cache__t *txn_cache;
590 /* the position where to reset it */
591 svn_cache__t **to_reset;
594 /* APR pool cleanup handler that will reset the cache pointer given in
597 remove_txn_cache(void *baton_void)
599 struct txn_cleanup_baton_t *baton = baton_void;
601 /* be careful not to hurt performance by resetting newer txn's caches. */
602 if (*baton->to_reset == baton->txn_cache)
604 /* This is equivalent to calling svn_fs_fs__reset_txn_caches(). */
605 *baton->to_reset = NULL;
611 /* This function sets / registers the required callbacks for a given
612 * transaction-specific *CACHE object, if CACHE is not NULL and a no-op
613 * otherwise. In particular, it will ensure that *CACHE gets reset to NULL
614 * upon POOL destruction latest.
617 init_txn_callbacks(svn_cache__t **cache,
622 struct txn_cleanup_baton_t *baton;
624 baton = apr_palloc(pool, sizeof(*baton));
625 baton->txn_cache = *cache;
626 baton->to_reset = cache;
628 apr_pool_cleanup_register(pool,
631 apr_pool_cleanup_null);
636 svn_fs_fs__initialize_txn_caches(svn_fs_t *fs,
640 fs_fs_data_t *ffd = fs->fsap_data;
642 /* Transaction content needs to be carefully prefixed to virtually
643 eliminate any chance for conflicts. The (repo, txn_id) pair
644 should be unique but if a transaction fails, it might be possible
645 to start a new transaction later that receives the same id.
646 Therefore, throw in a uuid as well - just to be sure. */
647 const char *prefix = apr_pstrcat(pool,
651 ":", svn_uuid_generate(pool), ":",
654 /* We don't support caching for concurrent transactions in the SAME
655 * FSFS session. Maybe, you forgot to clean POOL. */
656 if (ffd->txn_dir_cache != NULL || ffd->concurrent_transactions)
658 ffd->txn_dir_cache = NULL;
659 ffd->concurrent_transactions = TRUE;
664 /* create a txn-local directory cache */
665 SVN_ERR(create_cache(&ffd->txn_dir_cache,
667 svn_cache__get_global_membuffer_cache(),
669 svn_fs_fs__serialize_dir_entries,
670 svn_fs_fs__deserialize_dir_entries,
672 apr_pstrcat(pool, prefix, "TXNDIR",
678 /* reset the transaction-specific cache if the pool gets cleaned up. */
679 init_txn_callbacks(&(ffd->txn_dir_cache), pool);
685 svn_fs_fs__reset_txn_caches(svn_fs_t *fs)
687 /* we can always just reset the caches. This may degrade performance but
688 * can never cause in incorrect behavior. */
690 fs_fs_data_t *ffd = fs->fsap_data;
691 ffd->txn_dir_cache = NULL;