/* caching.c : in-memory caching * * ==================================================================== * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * ==================================================================== */ #include "fs.h" #include "fs_fs.h" #include "id.h" #include "dag.h" #include "tree.h" #include "temp_serializer.h" #include "../libsvn_fs/fs-loader.h" #include "svn_config.h" #include "svn_cache_config.h" #include "svn_private_config.h" #include "svn_hash.h" #include "svn_pools.h" #include "private/svn_debug.h" #include "private/svn_subr_private.h" /* Take the ORIGINAL string and replace all occurrences of ":" without * limiting the key space. Allocate the result in POOL. */ static const char * normalize_key_part(const char *original, apr_pool_t *pool) { apr_size_t i; apr_size_t len = strlen(original); svn_stringbuf_t *normalized = svn_stringbuf_create_ensure(len, pool); for (i = 0; i < len; ++i) { char c = original[i]; switch (c) { case ':': svn_stringbuf_appendbytes(normalized, "%_", 2); break; case '%': svn_stringbuf_appendbytes(normalized, "%%", 2); break; default : svn_stringbuf_appendbyte(normalized, c); } } return normalized->data; } /* Return a memcache in *MEMCACHE_P for FS if it's configured to use memcached, or NULL otherwise. Also, sets *FAIL_STOP to a boolean indicating whether cache errors should be returned to the caller or just passed to the FS warning handler. *CACHE_TXDELTAS, *CACHE_FULLTEXTS and *CACHE_REVPROPS flags will be set according to FS->CONFIG. *CACHE_NAMESPACE receives the cache prefix to use. Use FS->pool for allocating the memcache and CACHE_NAMESPACE, and POOL for temporary allocations. */ static svn_error_t * read_config(svn_memcache_t **memcache_p, svn_boolean_t *fail_stop, const char **cache_namespace, svn_boolean_t *cache_txdeltas, svn_boolean_t *cache_fulltexts, svn_boolean_t *cache_revprops, svn_fs_t *fs, apr_pool_t *pool) { fs_fs_data_t *ffd = fs->fsap_data; SVN_ERR(svn_cache__make_memcache_from_config(memcache_p, ffd->config, fs->pool)); /* No cache namespace by default. I.e. all FS instances share the * cached data. If you specify different namespaces, the data will * share / compete for the same cache memory but keys will not match * across namespaces and, thus, cached data will not be shared between * namespaces. * * Since the namespace will be concatenated with other elements to form * the complete key prefix, we must make sure that the resulting string * is unique and cannot be created by any other combination of elements. */ *cache_namespace = normalize_key_part(svn_hash__get_cstring(fs->config, SVN_FS_CONFIG_FSFS_CACHE_NS, ""), pool); /* don't cache text deltas by default. * Once we reconstructed the fulltexts from the deltas, * these deltas are rarely re-used. Therefore, only tools * like svnadmin will activate this to speed up operations * dump and verify. */ *cache_txdeltas = svn_hash__get_bool(fs->config, SVN_FS_CONFIG_FSFS_CACHE_DELTAS, FALSE); /* by default, cache fulltexts. * Most SVN tools care about reconstructed file content. * Thus, this is a reasonable default. * SVN admin tools may set that to FALSE because fulltexts * won't be re-used rendering the cache less effective * by squeezing wanted data out. */ *cache_fulltexts = svn_hash__get_bool(fs->config, SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS, TRUE); /* For now, always disable revprop caching. */ *cache_revprops = FALSE; return svn_config_get_bool(ffd->config, fail_stop, CONFIG_SECTION_CACHES, CONFIG_OPTION_FAIL_STOP, FALSE); } /* Implements svn_cache__error_handler_t * This variant clears the error after logging it. */ static svn_error_t * warn_and_continue_on_cache_errors(svn_error_t *err, void *baton, apr_pool_t *pool) { svn_fs_t *fs = baton; (fs->warning)(fs->warning_baton, err); svn_error_clear(err); return SVN_NO_ERROR; } /* Implements svn_cache__error_handler_t * This variant logs the error and passes it on to the callers. */ static svn_error_t * warn_and_fail_on_cache_errors(svn_error_t *err, void *baton, apr_pool_t *pool) { svn_fs_t *fs = baton; (fs->warning)(fs->warning_baton, err); return err; } #ifdef SVN_DEBUG_CACHE_DUMP_STATS /* Baton to be used for the dump_cache_statistics() pool cleanup function, */ struct dump_cache_baton_t { /* the pool about to be cleaned up. Will be used for temp. allocations. */ apr_pool_t *pool; /* the cache to dump the statistics for */ svn_cache__t *cache; }; /* APR pool cleanup handler that will printf the statistics of the cache referenced by the baton in BATON_VOID. */ static apr_status_t dump_cache_statistics(void *baton_void) { struct dump_cache_baton_t *baton = baton_void; apr_status_t result = APR_SUCCESS; svn_cache__info_t info; svn_string_t *text_stats; apr_array_header_t *lines; int i; svn_error_t *err = svn_cache__get_info(baton->cache, &info, TRUE, baton->pool); if (! err) { text_stats = svn_cache__format_info(&info, baton->pool); lines = svn_cstring_split(text_stats->data, "\n", FALSE, baton->pool); for (i = 0; i < lines->nelts; ++i) { const char *line = APR_ARRAY_IDX(lines, i, const char *); #ifdef SVN_DEBUG SVN_DBG(("%s\n", line)); #endif } } /* process error returns */ if (err) { result = err->apr_err; svn_error_clear(err); } return result; } #endif /* SVN_DEBUG_CACHE_DUMP_STATS */ /* This function sets / registers the required callbacks for a given * not transaction-specific CACHE object in FS, if CACHE is not NULL. * * All these svn_cache__t instances shall be handled uniformly. Unless * ERROR_HANDLER is NULL, register it for the given CACHE in FS. */ static svn_error_t * init_callbacks(svn_cache__t *cache, svn_fs_t *fs, svn_cache__error_handler_t error_handler, apr_pool_t *pool) { if (cache != NULL) { #ifdef SVN_DEBUG_CACHE_DUMP_STATS /* schedule printing the access statistics upon pool cleanup, * i.e. end of FSFS session. */ struct dump_cache_baton_t *baton; baton = apr_palloc(pool, sizeof(*baton)); baton->pool = pool; baton->cache = cache; apr_pool_cleanup_register(pool, baton, dump_cache_statistics, apr_pool_cleanup_null); #endif if (error_handler) SVN_ERR(svn_cache__set_error_handler(cache, error_handler, fs, pool)); } return SVN_NO_ERROR; } /* Sets *CACHE_P to cache instance based on provided options. * Creates memcache if MEMCACHE is not NULL. Creates membuffer cache if * MEMBUFFER is not NULL. Fallbacks to inprocess cache if MEMCACHE and * MEMBUFFER are NULL and pages is non-zero. Sets *CACHE_P to NULL * otherwise. * * Unless NO_HANDLER is true, register an error handler that reports errors * as warnings to the FS warning callback. * * Cache is allocated in POOL. * */ static svn_error_t * create_cache(svn_cache__t **cache_p, svn_memcache_t *memcache, svn_membuffer_t *membuffer, apr_int64_t pages, apr_int64_t items_per_page, svn_cache__serialize_func_t serializer, svn_cache__deserialize_func_t deserializer, apr_ssize_t klen, const char *prefix, svn_fs_t *fs, svn_boolean_t no_handler, apr_pool_t *pool) { svn_cache__error_handler_t error_handler = no_handler ? NULL : warn_and_fail_on_cache_errors; if (memcache) { SVN_ERR(svn_cache__create_memcache(cache_p, memcache, serializer, deserializer, klen, prefix, pool)); error_handler = no_handler ? NULL : warn_and_continue_on_cache_errors; } else if (membuffer) { SVN_ERR(svn_cache__create_membuffer_cache( cache_p, membuffer, serializer, deserializer, klen, prefix, FALSE, pool)); } else if (pages) { SVN_ERR(svn_cache__create_inprocess( cache_p, serializer, deserializer, klen, pages, items_per_page, FALSE, prefix, pool)); } else { *cache_p = NULL; } SVN_ERR(init_callbacks(*cache_p, fs, error_handler, pool)); return SVN_NO_ERROR; } svn_error_t * svn_fs_fs__initialize_caches(svn_fs_t *fs, apr_pool_t *pool) { fs_fs_data_t *ffd = fs->fsap_data; const char *prefix = apr_pstrcat(pool, "fsfs:", fs->uuid, "/", normalize_key_part(fs->path, pool), ":", (char *)NULL); svn_memcache_t *memcache; svn_membuffer_t *membuffer; svn_boolean_t no_handler; svn_boolean_t cache_txdeltas; svn_boolean_t cache_fulltexts; svn_boolean_t cache_revprops; const char *cache_namespace; /* Evaluating the cache configuration. */ SVN_ERR(read_config(&memcache, &no_handler, &cache_namespace, &cache_txdeltas, &cache_fulltexts, &cache_revprops, fs, pool)); prefix = apr_pstrcat(pool, "ns:", cache_namespace, ":", prefix, NULL); membuffer = svn_cache__get_global_membuffer_cache(); /* Make the cache for revision roots. For the vast majority of * commands, this is only going to contain a few entries (svnadmin * dump/verify is an exception here), so to reduce overhead let's * try to keep it to just one page. I estimate each entry has about * 72 bytes of overhead (svn_revnum_t key, svn_fs_id_t + * id_private_t + 3 strings for value, and the cache_entry); the * default pool size is 8192, so about a hundred should fit * comfortably. */ SVN_ERR(create_cache(&(ffd->rev_root_id_cache), NULL, membuffer, 1, 100, svn_fs_fs__serialize_id, svn_fs_fs__deserialize_id, sizeof(svn_revnum_t), apr_pstrcat(pool, prefix, "RRI", (char *)NULL), fs, no_handler, fs->pool)); /* Rough estimate: revision DAG nodes have size around 320 bytes, so * let's put 16 on a page. */ SVN_ERR(create_cache(&(ffd->rev_node_cache), NULL, membuffer, 1024, 16, svn_fs_fs__dag_serialize, svn_fs_fs__dag_deserialize, APR_HASH_KEY_STRING, apr_pstrcat(pool, prefix, "DAG", (char *)NULL), fs, no_handler, fs->pool)); /* 1st level DAG node cache */ ffd->dag_node_cache = svn_fs_fs__create_dag_cache(pool); /* Very rough estimate: 1K per directory. */ SVN_ERR(create_cache(&(ffd->dir_cache), NULL, membuffer, 1024, 8, svn_fs_fs__serialize_dir_entries, svn_fs_fs__deserialize_dir_entries, APR_HASH_KEY_STRING, apr_pstrcat(pool, prefix, "DIR", (char *)NULL), fs, no_handler, fs->pool)); /* Only 16 bytes per entry (a revision number + the corresponding offset). Since we want ~8k pages, that means 512 entries per page. */ SVN_ERR(create_cache(&(ffd->packed_offset_cache), NULL, membuffer, 32, 1, svn_fs_fs__serialize_manifest, svn_fs_fs__deserialize_manifest, sizeof(svn_revnum_t), apr_pstrcat(pool, prefix, "PACK-MANIFEST", (char *)NULL), fs, no_handler, fs->pool)); /* initialize node revision cache, if caching has been enabled */ SVN_ERR(create_cache(&(ffd->node_revision_cache), NULL, membuffer, 0, 0, /* Do not use inprocess cache */ svn_fs_fs__serialize_node_revision, svn_fs_fs__deserialize_node_revision, sizeof(pair_cache_key_t), apr_pstrcat(pool, prefix, "NODEREVS", (char *)NULL), fs, no_handler, fs->pool)); /* initialize node change list cache, if caching has been enabled */ SVN_ERR(create_cache(&(ffd->changes_cache), NULL, membuffer, 0, 0, /* Do not use inprocess cache */ svn_fs_fs__serialize_changes, svn_fs_fs__deserialize_changes, sizeof(svn_revnum_t), apr_pstrcat(pool, prefix, "CHANGES", (char *)NULL), fs, no_handler, fs->pool)); /* if enabled, cache fulltext and other derived information */ if (cache_fulltexts) { SVN_ERR(create_cache(&(ffd->fulltext_cache), memcache, membuffer, 0, 0, /* Do not use inprocess cache */ /* Values are svn_stringbuf_t */ NULL, NULL, sizeof(pair_cache_key_t), apr_pstrcat(pool, prefix, "TEXT", (char *)NULL), fs, no_handler, fs->pool)); SVN_ERR(create_cache(&(ffd->properties_cache), NULL, membuffer, 0, 0, /* Do not use inprocess cache */ svn_fs_fs__serialize_properties, svn_fs_fs__deserialize_properties, sizeof(pair_cache_key_t), apr_pstrcat(pool, prefix, "PROP", (char *)NULL), fs, no_handler, fs->pool)); SVN_ERR(create_cache(&(ffd->mergeinfo_cache), NULL, membuffer, 0, 0, /* Do not use inprocess cache */ svn_fs_fs__serialize_mergeinfo, svn_fs_fs__deserialize_mergeinfo, APR_HASH_KEY_STRING, apr_pstrcat(pool, prefix, "MERGEINFO", (char *)NULL), fs, no_handler, fs->pool)); SVN_ERR(create_cache(&(ffd->mergeinfo_existence_cache), NULL, membuffer, 0, 0, /* Do not use inprocess cache */ /* Values are svn_stringbuf_t */ NULL, NULL, APR_HASH_KEY_STRING, apr_pstrcat(pool, prefix, "HAS_MERGEINFO", (char *)NULL), fs, no_handler, fs->pool)); } else { ffd->fulltext_cache = NULL; ffd->properties_cache = NULL; ffd->mergeinfo_cache = NULL; ffd->mergeinfo_existence_cache = NULL; } /* initialize revprop cache, if full-text caching has been enabled */ if (cache_revprops) { SVN_ERR(create_cache(&(ffd->revprop_cache), NULL, membuffer, 0, 0, /* Do not use inprocess cache */ svn_fs_fs__serialize_properties, svn_fs_fs__deserialize_properties, sizeof(pair_cache_key_t), apr_pstrcat(pool, prefix, "REVPROP", (char *)NULL), fs, no_handler, fs->pool)); } else { ffd->revprop_cache = NULL; } /* if enabled, cache text deltas and their combinations */ if (cache_txdeltas) { SVN_ERR(create_cache(&(ffd->txdelta_window_cache), NULL, membuffer, 0, 0, /* Do not use inprocess cache */ svn_fs_fs__serialize_txdelta_window, svn_fs_fs__deserialize_txdelta_window, APR_HASH_KEY_STRING, apr_pstrcat(pool, prefix, "TXDELTA_WINDOW", (char *)NULL), fs, no_handler, fs->pool)); SVN_ERR(create_cache(&(ffd->combined_window_cache), NULL, membuffer, 0, 0, /* Do not use inprocess cache */ /* Values are svn_stringbuf_t */ NULL, NULL, APR_HASH_KEY_STRING, apr_pstrcat(pool, prefix, "COMBINED_WINDOW", (char *)NULL), fs, no_handler, fs->pool)); } else { ffd->txdelta_window_cache = NULL; ffd->combined_window_cache = NULL; } return SVN_NO_ERROR; } /* Baton to be used for the remove_txn_cache() pool cleanup function, */ struct txn_cleanup_baton_t { /* the cache to reset */ svn_cache__t *txn_cache; /* the position where to reset it */ svn_cache__t **to_reset; }; /* APR pool cleanup handler that will reset the cache pointer given in BATON_VOID. */ static apr_status_t remove_txn_cache(void *baton_void) { struct txn_cleanup_baton_t *baton = baton_void; /* be careful not to hurt performance by resetting newer txn's caches. */ if (*baton->to_reset == baton->txn_cache) { /* This is equivalent to calling svn_fs_fs__reset_txn_caches(). */ *baton->to_reset = NULL; } return APR_SUCCESS; } /* This function sets / registers the required callbacks for a given * transaction-specific *CACHE object, if CACHE is not NULL and a no-op * otherwise. In particular, it will ensure that *CACHE gets reset to NULL * upon POOL destruction latest. */ static void init_txn_callbacks(svn_cache__t **cache, apr_pool_t *pool) { if (*cache != NULL) { struct txn_cleanup_baton_t *baton; baton = apr_palloc(pool, sizeof(*baton)); baton->txn_cache = *cache; baton->to_reset = cache; apr_pool_cleanup_register(pool, baton, remove_txn_cache, apr_pool_cleanup_null); } } svn_error_t * svn_fs_fs__initialize_txn_caches(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool) { fs_fs_data_t *ffd = fs->fsap_data; /* Transaction content needs to be carefully prefixed to virtually eliminate any chance for conflicts. The (repo, txn_id) pair should be unique but if a transaction fails, it might be possible to start a new transaction later that receives the same id. Therefore, throw in a uuid as well - just to be sure. */ const char *prefix = apr_pstrcat(pool, "fsfs:", fs->uuid, "/", fs->path, ":", txn_id, ":", svn_uuid_generate(pool), ":", (char *)NULL); /* We don't support caching for concurrent transactions in the SAME * FSFS session. Maybe, you forgot to clean POOL. */ if (ffd->txn_dir_cache != NULL || ffd->concurrent_transactions) { ffd->txn_dir_cache = NULL; ffd->concurrent_transactions = TRUE; return SVN_NO_ERROR; } /* create a txn-local directory cache */ SVN_ERR(create_cache(&ffd->txn_dir_cache, NULL, svn_cache__get_global_membuffer_cache(), 1024, 8, svn_fs_fs__serialize_dir_entries, svn_fs_fs__deserialize_dir_entries, APR_HASH_KEY_STRING, apr_pstrcat(pool, prefix, "TXNDIR", (char *)NULL), fs, TRUE, pool)); /* reset the transaction-specific cache if the pool gets cleaned up. */ init_txn_callbacks(&(ffd->txn_dir_cache), pool); return SVN_NO_ERROR; } void svn_fs_fs__reset_txn_caches(svn_fs_t *fs) { /* we can always just reset the caches. This may degrade performance but * can never cause in incorrect behavior. */ fs_fs_data_t *ffd = fs->fsap_data; ffd->txn_dir_cache = NULL; }