1 /* caching.c : in-memory caching
3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
20 * ====================================================================
31 #include "temp_serializer.h"
33 #include "../libsvn_fs/fs-loader.h"
35 #include "svn_config.h"
36 #include "svn_cache_config.h"
38 #include "svn_private_config.h"
40 #include "svn_pools.h"
42 #include "private/svn_debug.h"
43 #include "private/svn_subr_private.h"
45 /* Take the ORIGINAL string and replace all occurrences of ":" without
46 * limiting the key space. Allocate the result in RESULT_POOL.
49 normalize_key_part(const char *original,
50 apr_pool_t *result_pool)
53 apr_size_t len = strlen(original);
54 svn_stringbuf_t *normalized = svn_stringbuf_create_ensure(len,
57 for (i = 0; i < len; ++i)
62 case ':': svn_stringbuf_appendbytes(normalized, "%_", 2);
64 case '%': svn_stringbuf_appendbytes(normalized, "%%", 2);
66 default : svn_stringbuf_appendbyte(normalized, c);
70 return normalized->data;
73 /* *CACHE_TXDELTAS, *CACHE_FULLTEXTS and *CACHE_REVPROPS flags will be set
74 according to FS->CONFIG. *CACHE_NAMESPACE receives the cache prefix
77 Allocate CACHE_NAMESPACE in RESULT_POOL. */
79 read_config(const char **cache_namespace,
80 svn_boolean_t *cache_txdeltas,
81 svn_boolean_t *cache_fulltexts,
82 svn_boolean_t *cache_revprops,
84 apr_pool_t *result_pool)
86 /* No cache namespace by default. I.e. all FS instances share the
87 * cached data. If you specify different namespaces, the data will
88 * share / compete for the same cache memory but keys will not match
89 * across namespaces and, thus, cached data will not be shared between
92 * Since the namespace will be concatenated with other elements to form
93 * the complete key prefix, we must make sure that the resulting string
94 * is unique and cannot be created by any other combination of elements.
97 = normalize_key_part(svn_hash__get_cstring(fs->config,
98 SVN_FS_CONFIG_FSFS_CACHE_NS,
102 /* don't cache text deltas by default.
103 * Once we reconstructed the fulltexts from the deltas,
104 * these deltas are rarely re-used. Therefore, only tools
105 * like svnadmin will activate this to speed up operations
109 = svn_hash__get_bool(fs->config,
110 SVN_FS_CONFIG_FSFS_CACHE_DELTAS,
113 /* by default, cache fulltexts.
114 * Most SVN tools care about reconstructed file content.
115 * Thus, this is a reasonable default.
116 * SVN admin tools may set that to FALSE because fulltexts
117 * won't be re-used rendering the cache less effective
118 * by squeezing wanted data out.
121 = svn_hash__get_bool(fs->config,
122 SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS,
125 /* don't cache revprops by default.
126 * Revprop caching significantly speeds up operations like
127 * svn ls -v. However, it requires synchronization that may
128 * not be available or efficient in the current server setup.
129 * Option "2" is equivalent to "1".
131 if (strcmp(svn_hash__get_cstring(fs->config,
132 SVN_FS_CONFIG_FSFS_CACHE_REVPROPS,
135 = svn_hash__get_bool(fs->config,
136 SVN_FS_CONFIG_FSFS_CACHE_REVPROPS,
139 *cache_revprops = TRUE;
145 /* Implements svn_cache__error_handler_t
146 * This variant clears the error after logging it.
149 warn_and_continue_on_cache_errors(svn_error_t *err,
153 svn_fs_t *fs = baton;
154 (fs->warning)(fs->warning_baton, err);
155 svn_error_clear(err);
160 /* Implements svn_cache__error_handler_t
161 * This variant logs the error and passes it on to the callers.
164 warn_and_fail_on_cache_errors(svn_error_t *err,
168 svn_fs_t *fs = baton;
169 (fs->warning)(fs->warning_baton, err);
173 #ifdef SVN_DEBUG_CACHE_DUMP_STATS
174 /* Baton to be used for the dump_cache_statistics() pool cleanup function, */
175 typedef struct dump_cache_baton_t
177 /* the pool about to be cleaned up. Will be used for temp. allocations. */
180 /* the cache to dump the statistics for */
182 } dump_cache_baton_t;
184 /* APR pool cleanup handler that will printf the statistics of the
185 cache referenced by the baton in BATON_VOID. */
187 dump_cache_statistics(void *baton_void)
189 dump_cache_baton_t *baton = baton_void;
191 apr_status_t result = APR_SUCCESS;
192 svn_cache__info_t info;
193 svn_string_t *text_stats;
194 apr_array_header_t *lines;
197 svn_error_t *err = svn_cache__get_info(baton->cache,
202 /* skip unused caches */
203 if (! err && (info.gets > 0 || info.sets > 0))
205 text_stats = svn_cache__format_info(&info, TRUE, baton->pool);
206 lines = svn_cstring_split(text_stats->data, "\n", FALSE, baton->pool);
208 for (i = 0; i < lines->nelts; ++i)
210 const char *line = APR_ARRAY_IDX(lines, i, const char *);
212 SVN_DBG(("%s\n", line));
217 /* process error returns */
220 result = err->apr_err;
221 svn_error_clear(err);
228 dump_global_cache_statistics(void *baton_void)
230 apr_pool_t *pool = baton_void;
232 svn_cache__info_t *info = svn_cache__membuffer_get_global_info(pool);
233 svn_string_t *text_stats = svn_cache__format_info(info, FALSE, pool);
234 apr_array_header_t *lines = svn_cstring_split(text_stats->data, "\n",
238 for (i = 0; i < lines->nelts; ++i)
240 const char *line = APR_ARRAY_IDX(lines, i, const char *);
242 SVN_DBG(("%s\n", line));
249 #endif /* SVN_DEBUG_CACHE_DUMP_STATS */
251 /* This function sets / registers the required callbacks for a given
252 * not transaction-specific CACHE object in FS, if CACHE is not NULL.
254 * All these svn_cache__t instances shall be handled uniformly. Unless
255 * ERROR_HANDLER is NULL, register it for the given CACHE in FS.
258 init_callbacks(svn_cache__t *cache,
260 svn_cache__error_handler_t error_handler,
265 #ifdef SVN_DEBUG_CACHE_DUMP_STATS
267 /* schedule printing the access statistics upon pool cleanup,
268 * i.e. end of FSX session.
270 dump_cache_baton_t *baton;
272 baton = apr_palloc(pool, sizeof(*baton));
274 baton->cache = cache;
276 apr_pool_cleanup_register(pool,
278 dump_cache_statistics,
279 apr_pool_cleanup_null);
283 SVN_ERR(svn_cache__set_error_handler(cache,
293 /* Sets *CACHE_P to cache instance based on provided options.
294 * Creates memcache if MEMCACHE is not NULL. Creates membuffer cache if
295 * MEMBUFFER is not NULL. Fallbacks to inprocess cache if MEMCACHE and
296 * MEMBUFFER are NULL and pages is non-zero. Sets *CACHE_P to NULL
297 * otherwise. Use the given PRIORITY class for the new cache. If it
298 * is 0, then use the default priority class.
300 * Unless NO_HANDLER is true, register an error handler that reports errors
301 * as warnings to the FS warning callback.
303 * Cache is allocated in RESULT_POOL, temporaries in SCRATCH_POOL.
306 create_cache(svn_cache__t **cache_p,
307 svn_memcache_t *memcache,
308 svn_membuffer_t *membuffer,
310 apr_int64_t items_per_page,
311 svn_cache__serialize_func_t serializer,
312 svn_cache__deserialize_func_t deserializer,
315 apr_uint32_t priority,
317 svn_boolean_t no_handler,
318 apr_pool_t *result_pool,
319 apr_pool_t *scratch_pool)
321 svn_cache__error_handler_t error_handler = no_handler
323 : warn_and_fail_on_cache_errors;
325 priority = SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY;
329 SVN_ERR(svn_cache__create_memcache(cache_p, memcache,
330 serializer, deserializer, klen,
331 prefix, result_pool));
332 error_handler = no_handler
334 : warn_and_continue_on_cache_errors;
338 SVN_ERR(svn_cache__create_membuffer_cache(
339 cache_p, membuffer, serializer, deserializer,
340 klen, prefix, priority, FALSE, result_pool, scratch_pool));
344 SVN_ERR(svn_cache__create_inprocess(
345 cache_p, serializer, deserializer, klen, pages,
346 items_per_page, FALSE, prefix, result_pool));
353 SVN_ERR(init_callbacks(*cache_p, fs, error_handler, result_pool));
359 svn_fs_x__initialize_caches(svn_fs_t *fs,
360 apr_pool_t *scratch_pool)
362 svn_fs_x__data_t *ffd = fs->fsap_data;
363 const char *prefix = apr_pstrcat(scratch_pool,
365 "/", normalize_key_part(fs->path,
369 svn_membuffer_t *membuffer;
370 svn_boolean_t no_handler = ffd->fail_stop;
371 svn_boolean_t cache_txdeltas;
372 svn_boolean_t cache_fulltexts;
373 svn_boolean_t cache_revprops;
374 const char *cache_namespace;
376 /* Evaluating the cache configuration. */
377 SVN_ERR(read_config(&cache_namespace,
384 prefix = apr_pstrcat(scratch_pool, "ns:", cache_namespace, ":", prefix,
387 membuffer = svn_cache__get_global_membuffer_cache();
389 /* General rules for assigning cache priorities:
391 * - Data that can be reconstructed from other elements has low prio
392 * (e.g. fulltexts, directories etc.)
393 * - Index data required to find any of the other data has high prio
394 * (e.g. noderevs, L2P and P2L index pages)
395 * - everthing else should use default prio
398 #ifdef SVN_DEBUG_CACHE_DUMP_STATS
400 /* schedule printing the global access statistics upon pool cleanup,
401 * i.e. end of FSX session.
404 apr_pool_cleanup_register(fs->pool,
406 dump_global_cache_statistics,
407 apr_pool_cleanup_null);
410 /* Rough estimate: revision DAG nodes have size around 320 bytes, so
411 * let's put 16 on a page. */
412 SVN_ERR(create_cache(&(ffd->rev_node_cache),
416 svn_fs_x__dag_serialize,
417 svn_fs_x__dag_deserialize,
419 apr_pstrcat(scratch_pool, prefix, "DAG", SVN_VA_NULL),
420 SVN_CACHE__MEMBUFFER_LOW_PRIORITY,
423 fs->pool, scratch_pool));
425 /* 1st level DAG node cache */
426 ffd->dag_node_cache = svn_fs_x__create_dag_cache(fs->pool);
428 /* Very rough estimate: 1K per directory. */
429 SVN_ERR(create_cache(&(ffd->dir_cache),
433 svn_fs_x__serialize_dir_entries,
434 svn_fs_x__deserialize_dir_entries,
435 sizeof(svn_fs_x__id_t),
436 apr_pstrcat(scratch_pool, prefix, "DIR", SVN_VA_NULL),
437 SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
440 fs->pool, scratch_pool));
442 /* Only 16 bytes per entry (a revision number + the corresponding offset).
443 Since we want ~8k pages, that means 512 entries per page. */
444 SVN_ERR(create_cache(&(ffd->packed_offset_cache),
448 svn_fs_x__serialize_manifest,
449 svn_fs_x__deserialize_manifest,
450 sizeof(svn_revnum_t),
451 apr_pstrcat(scratch_pool, prefix, "PACK-MANIFEST",
453 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
456 fs->pool, scratch_pool));
458 /* initialize node revision cache, if caching has been enabled */
459 SVN_ERR(create_cache(&(ffd->node_revision_cache),
462 32, 32, /* ~200 byte / entry; 1k entries total */
463 svn_fs_x__serialize_node_revision,
464 svn_fs_x__deserialize_node_revision,
465 sizeof(svn_fs_x__pair_cache_key_t),
466 apr_pstrcat(scratch_pool, prefix, "NODEREVS",
468 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
471 fs->pool, scratch_pool));
473 /* initialize representation header cache, if caching has been enabled */
474 SVN_ERR(create_cache(&(ffd->rep_header_cache),
477 1, 1000, /* ~8 bytes / entry; 1k entries total */
478 svn_fs_x__serialize_rep_header,
479 svn_fs_x__deserialize_rep_header,
480 sizeof(svn_fs_x__representation_cache_key_t),
481 apr_pstrcat(scratch_pool, prefix, "REPHEADER",
483 SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
486 fs->pool, scratch_pool));
488 /* initialize node change list cache, if caching has been enabled */
489 SVN_ERR(create_cache(&(ffd->changes_cache),
492 1, 8, /* 1k / entry; 8 entries total, rarely used */
493 svn_fs_x__serialize_changes,
494 svn_fs_x__deserialize_changes,
495 sizeof(svn_revnum_t),
496 apr_pstrcat(scratch_pool, prefix, "CHANGES",
501 fs->pool, scratch_pool));
503 /* if enabled, cache fulltext and other derived information */
506 SVN_ERR(create_cache(&(ffd->fulltext_cache),
509 0, 0, /* Do not use inprocess cache */
510 /* Values are svn_stringbuf_t */
512 sizeof(svn_fs_x__pair_cache_key_t),
513 apr_pstrcat(scratch_pool, prefix, "TEXT",
515 SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
518 fs->pool, scratch_pool));
520 SVN_ERR(create_cache(&(ffd->properties_cache),
523 0, 0, /* Do not use inprocess cache */
524 svn_fs_x__serialize_properties,
525 svn_fs_x__deserialize_properties,
526 sizeof(svn_fs_x__pair_cache_key_t),
527 apr_pstrcat(scratch_pool, prefix, "PROP",
529 SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
532 fs->pool, scratch_pool));
534 SVN_ERR(create_cache(&(ffd->mergeinfo_cache),
537 0, 0, /* Do not use inprocess cache */
538 svn_fs_x__serialize_mergeinfo,
539 svn_fs_x__deserialize_mergeinfo,
541 apr_pstrcat(scratch_pool, prefix, "MERGEINFO",
546 fs->pool, scratch_pool));
548 SVN_ERR(create_cache(&(ffd->mergeinfo_existence_cache),
551 0, 0, /* Do not use inprocess cache */
552 /* Values are svn_stringbuf_t */
555 apr_pstrcat(scratch_pool, prefix, "HAS_MERGEINFO",
560 fs->pool, scratch_pool));
564 ffd->fulltext_cache = NULL;
565 ffd->properties_cache = NULL;
566 ffd->mergeinfo_cache = NULL;
567 ffd->mergeinfo_existence_cache = NULL;
570 /* if enabled, cache revprops */
573 SVN_ERR(create_cache(&(ffd->revprop_cache),
576 0, 0, /* Do not use inprocess cache */
577 svn_fs_x__serialize_properties,
578 svn_fs_x__deserialize_properties,
579 sizeof(svn_fs_x__pair_cache_key_t),
580 apr_pstrcat(scratch_pool, prefix, "REVPROP",
582 SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
585 fs->pool, scratch_pool));
589 ffd->revprop_cache = NULL;
592 /* if enabled, cache text deltas and their combinations */
595 SVN_ERR(create_cache(&(ffd->txdelta_window_cache),
598 0, 0, /* Do not use inprocess cache */
599 svn_fs_x__serialize_txdelta_window,
600 svn_fs_x__deserialize_txdelta_window,
601 sizeof(svn_fs_x__window_cache_key_t),
602 apr_pstrcat(scratch_pool, prefix, "TXDELTA_WINDOW",
604 SVN_CACHE__MEMBUFFER_LOW_PRIORITY,
607 fs->pool, scratch_pool));
609 SVN_ERR(create_cache(&(ffd->combined_window_cache),
612 0, 0, /* Do not use inprocess cache */
613 /* Values are svn_stringbuf_t */
615 sizeof(svn_fs_x__window_cache_key_t),
616 apr_pstrcat(scratch_pool, prefix, "COMBINED_WINDOW",
618 SVN_CACHE__MEMBUFFER_LOW_PRIORITY,
621 fs->pool, scratch_pool));
625 ffd->txdelta_window_cache = NULL;
626 ffd->combined_window_cache = NULL;
629 SVN_ERR(create_cache(&(ffd->noderevs_container_cache),
632 16, 4, /* Important, largish objects */
633 svn_fs_x__serialize_noderevs_container,
634 svn_fs_x__deserialize_noderevs_container,
635 sizeof(svn_fs_x__pair_cache_key_t),
636 apr_pstrcat(scratch_pool, prefix, "NODEREVSCNT",
638 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
641 fs->pool, scratch_pool));
642 SVN_ERR(create_cache(&(ffd->changes_container_cache),
645 0, 0, /* Do not use inprocess cache */
646 svn_fs_x__serialize_changes_container,
647 svn_fs_x__deserialize_changes_container,
648 sizeof(svn_fs_x__pair_cache_key_t),
649 apr_pstrcat(scratch_pool, prefix, "CHANGESCNT",
654 fs->pool, scratch_pool));
655 SVN_ERR(create_cache(&(ffd->reps_container_cache),
658 0, 0, /* Do not use inprocess cache */
659 svn_fs_x__serialize_reps_container,
660 svn_fs_x__deserialize_reps_container,
661 sizeof(svn_fs_x__pair_cache_key_t),
662 apr_pstrcat(scratch_pool, prefix, "REPSCNT",
667 fs->pool, scratch_pool));
669 SVN_ERR(create_cache(&(ffd->l2p_header_cache),
672 64, 16, /* entry size varies but we must cover
673 a reasonable number of revisions (1k) */
674 svn_fs_x__serialize_l2p_header,
675 svn_fs_x__deserialize_l2p_header,
676 sizeof(svn_fs_x__pair_cache_key_t),
677 apr_pstrcat(scratch_pool, prefix, "L2P_HEADER",
679 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
682 fs->pool, scratch_pool));
683 SVN_ERR(create_cache(&(ffd->l2p_page_cache),
686 64, 16, /* entry size varies but we must cover
687 a reasonable number of revisions (1k) */
688 svn_fs_x__serialize_l2p_page,
689 svn_fs_x__deserialize_l2p_page,
690 sizeof(svn_fs_x__page_cache_key_t),
691 apr_pstrcat(scratch_pool, prefix, "L2P_PAGE",
693 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
696 fs->pool, scratch_pool));
697 SVN_ERR(create_cache(&(ffd->p2l_header_cache),
700 4, 1, /* Large entries. Rarely used. */
701 svn_fs_x__serialize_p2l_header,
702 svn_fs_x__deserialize_p2l_header,
703 sizeof(svn_fs_x__pair_cache_key_t),
704 apr_pstrcat(scratch_pool, prefix, "P2L_HEADER",
706 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
709 fs->pool, scratch_pool));
710 SVN_ERR(create_cache(&(ffd->p2l_page_cache),
713 4, 16, /* Variably sized entries. Rarely used. */
714 svn_fs_x__serialize_p2l_page,
715 svn_fs_x__deserialize_p2l_page,
716 sizeof(svn_fs_x__page_cache_key_t),
717 apr_pstrcat(scratch_pool, prefix, "P2L_PAGE",
719 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
722 fs->pool, scratch_pool));