2 * cache-memcache.c: memcached caching for Subversion
4 * ====================================================================
5 * Licensed to the Apache Software Foundation (ASF) under one
6 * or more contributor license agreements. See the NOTICE file
7 * distributed with this work for additional information
8 * regarding copyright ownership. The ASF licenses this file
9 * to you under the Apache License, Version 2.0 (the
10 * "License"); you may not use this file except in compliance
11 * with the License. You may obtain a copy of the License at
13 * http://www.apache.org/licenses/LICENSE-2.0
15 * Unless required by applicable law or agreed to in writing,
16 * software distributed under the License is distributed on an
17 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
18 * KIND, either express or implied. See the License for the
19 * specific language governing permissions and limitations
21 * ====================================================================
26 #include "svn_pools.h"
27 #include "svn_base64.h"
30 #include "svn_private_config.h"
31 #include "private/svn_cache.h"
32 #include "private/svn_dep_compat.h"
36 #ifdef SVN_HAVE_MEMCACHE
38 #include <apr_memcache.h>
40 /* A note on thread safety:
42 The apr_memcache_t object does its own mutex handling, and nothing
43 else in memcache_t is ever modified, so this implementation should
47 /* The (internal) cache object. */
48 typedef struct memcache_t {
49 /* The memcached server set we're using. */
50 apr_memcache_t *memcache;
52 /* A prefix used to differentiate our data from any other data in
53 * the memcached (URI-encoded). */
56 /* The size of the key: either a fixed number of bytes or
57 * APR_HASH_KEY_STRING. */
61 /* Used to marshal values in and out of the cache. */
62 svn_cache__serialize_func_t serialize_func;
63 svn_cache__deserialize_func_t deserialize_func;
66 /* The wrapper around apr_memcache_t. */
67 struct svn_memcache_t {
72 /* The memcached protocol says the maximum key length is 250. Let's
73 just say 249, to be safe. */
74 #define MAX_MEMCACHED_KEY_LEN 249
75 #define MEMCACHED_KEY_UNHASHED_LEN (MAX_MEMCACHED_KEY_LEN - \
76 2 * APR_MD5_DIGESTSIZE)
79 /* Set *MC_KEY to a memcache key for the given key KEY for CACHE, allocated
82 build_key(const char **mc_key,
87 const char *encoded_suffix;
89 apr_size_t long_key_len;
91 if (cache->klen == APR_HASH_KEY_STRING)
92 encoded_suffix = svn_path_uri_encode(raw_key, pool);
95 const svn_string_t *raw = svn_string_ncreate(raw_key, cache->klen, pool);
96 const svn_string_t *encoded = svn_base64_encode_string2(raw, FALSE,
98 encoded_suffix = encoded->data;
101 long_key = apr_pstrcat(pool, "SVN:", cache->prefix, ":", encoded_suffix,
103 long_key_len = strlen(long_key);
105 /* We don't want to have a key that's too big. If it was going to
106 be too big, we MD5 the entire string, then replace the last bit
107 with the checksum. Note that APR_MD5_DIGESTSIZE is for the pure
108 binary digest; we have to double that when we convert to hex.
110 Every key we use will either be at most
111 MEMCACHED_KEY_UNHASHED_LEN bytes long, or be exactly
112 MAX_MEMCACHED_KEY_LEN bytes long. */
113 if (long_key_len > MEMCACHED_KEY_UNHASHED_LEN)
115 svn_checksum_t *checksum;
116 SVN_ERR(svn_checksum(&checksum, svn_checksum_md5, long_key, long_key_len,
119 long_key = apr_pstrcat(pool,
120 apr_pstrmemdup(pool, long_key,
121 MEMCACHED_KEY_UNHASHED_LEN),
122 svn_checksum_to_cstring_display(checksum, pool),
130 /* Core functionality of our getter functions: fetch DATA from the memcached
131 * given by CACHE_VOID and identified by KEY. Indicate success in FOUND and
132 * use a tempoary sub-pool of POOL for allocations.
135 memcache_internal_get(char **data,
137 svn_boolean_t *found,
142 memcache_t *cache = cache_void;
143 apr_status_t apr_err;
153 subpool = svn_pool_create(pool);
154 SVN_ERR(build_key(&mc_key, cache, key, subpool));
156 apr_err = apr_memcache_getp(cache->memcache,
161 NULL /* ignore flags */);
162 if (apr_err == APR_NOTFOUND)
165 svn_pool_destroy(subpool);
168 else if (apr_err != APR_SUCCESS || !*data)
169 return svn_error_wrap_apr(apr_err,
170 _("Unknown memcached error while reading"));
174 svn_pool_destroy(subpool);
180 memcache_get(void **value_p,
181 svn_boolean_t *found,
184 apr_pool_t *result_pool)
186 memcache_t *cache = cache_void;
189 SVN_ERR(memcache_internal_get(&data,
196 /* If we found it, de-serialize it. */
199 if (cache->deserialize_func)
201 SVN_ERR((cache->deserialize_func)(value_p, data, data_len,
206 svn_string_t *value = apr_pcalloc(result_pool, sizeof(*value));
208 value->len = data_len;
216 /* Core functionality of our setter functions: store LENGH bytes of DATA
217 * to be identified by KEY in the memcached given by CACHE_VOID. Use POOL
218 * for temporary allocations.
221 memcache_internal_set(void *cache_void,
225 apr_pool_t *scratch_pool)
227 memcache_t *cache = cache_void;
229 apr_status_t apr_err;
231 SVN_ERR(build_key(&mc_key, cache, key, scratch_pool));
232 apr_err = apr_memcache_set(cache->memcache, mc_key, (char *)data, len, 0, 0);
234 /* ### Maybe write failures should be ignored (but logged)? */
235 if (apr_err != APR_SUCCESS)
236 return svn_error_wrap_apr(apr_err,
237 _("Unknown memcached error while writing"));
244 memcache_set(void *cache_void,
247 apr_pool_t *scratch_pool)
249 memcache_t *cache = cache_void;
250 apr_pool_t *subpool = svn_pool_create(scratch_pool);
258 if (cache->serialize_func)
260 SVN_ERR((cache->serialize_func)(&data, &data_len, value, subpool));
264 svn_stringbuf_t *value_str = value;
265 data = value_str->data;
266 data_len = value_str->len;
269 err = memcache_internal_set(cache_void, key, data, data_len, subpool);
271 svn_pool_destroy(subpool);
276 memcache_get_partial(void **value_p,
277 svn_boolean_t *found,
280 svn_cache__partial_getter_func_t func,
282 apr_pool_t *result_pool)
284 svn_error_t *err = SVN_NO_ERROR;
288 SVN_ERR(memcache_internal_get(&data,
295 /* If we found it, de-serialize it. */
297 ? func(value_p, data, size, baton, result_pool)
303 memcache_set_partial(void *cache_void,
305 svn_cache__partial_setter_func_t func,
307 apr_pool_t *scratch_pool)
309 svn_error_t *err = SVN_NO_ERROR;
313 svn_boolean_t found = FALSE;
315 apr_pool_t *subpool = svn_pool_create(scratch_pool);
316 SVN_ERR(memcache_internal_get((char **)&data,
323 /* If we found it, modify it and write it back to cache */
326 SVN_ERR(func(&data, &size, baton, subpool));
327 err = memcache_internal_set(cache_void, key, data, size, subpool);
330 svn_pool_destroy(subpool);
336 memcache_iter(svn_boolean_t *completed,
338 svn_iter_apr_hash_cb_t user_cb,
340 apr_pool_t *scratch_pool)
342 return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
343 _("Can't iterate a memcached cache"));
347 memcache_is_cachable(void *unused, apr_size_t size)
349 (void)unused; /* silence gcc warning. */
351 /* The memcached cutoff seems to be a bit (header length?) under a megabyte.
352 * We round down a little to be safe.
354 return size < 1000000;
358 memcache_get_info(void *cache_void,
359 svn_cache__info_t *info,
361 apr_pool_t *result_pool)
363 memcache_t *cache = cache_void;
365 info->id = apr_pstrdup(result_pool, cache->prefix);
367 /* we don't have any memory allocation info */
370 info->total_size = 0;
372 info->used_entries = 0;
373 info->total_entries = 0;
378 static svn_cache__vtable_t memcache_vtable = {
382 memcache_is_cachable,
383 memcache_get_partial,
384 memcache_set_partial,
389 svn_cache__create_memcache(svn_cache__t **cache_p,
390 svn_memcache_t *memcache,
391 svn_cache__serialize_func_t serialize_func,
392 svn_cache__deserialize_func_t deserialize_func,
397 svn_cache__t *wrapper = apr_pcalloc(pool, sizeof(*wrapper));
398 memcache_t *cache = apr_pcalloc(pool, sizeof(*cache));
400 cache->serialize_func = serialize_func;
401 cache->deserialize_func = deserialize_func;
403 cache->prefix = svn_path_uri_encode(prefix, pool);
404 cache->memcache = memcache->c;
406 wrapper->vtable = &memcache_vtable;
407 wrapper->cache_internal = cache;
408 wrapper->error_handler = 0;
409 wrapper->error_baton = 0;
416 /*** Creating apr_memcache_t from svn_config_t. ***/
418 /* Baton for add_memcache_server. */
420 apr_memcache_t *memcache;
421 apr_pool_t *memcache_pool;
425 /* Implements svn_config_enumerator2_t. */
427 add_memcache_server(const char *name,
432 struct ams_baton *b = baton;
435 apr_status_t apr_err;
436 apr_memcache_server_t *server;
438 apr_err = apr_parse_addr_port(&host, &scope, &port,
440 if (apr_err != APR_SUCCESS)
442 b->err = svn_error_wrap_apr(apr_err,
443 _("Error parsing memcache server '%s'"),
450 b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL,
451 _("Scope not allowed in memcache server "
458 b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL,
459 _("Must specify host and port for memcache "
465 /* Note: the four numbers here are only relevant when an
466 apr_memcache_t is being shared by multiple threads. */
467 apr_err = apr_memcache_server_create(b->memcache_pool,
470 0, /* min connections */
471 5, /* soft max connections */
472 10, /* hard max connections */
473 /* time to live (in microseconds) */
474 apr_time_from_sec(50),
476 if (apr_err != APR_SUCCESS)
478 b->err = svn_error_wrap_apr(apr_err,
479 _("Unknown error creating memcache server"));
483 apr_err = apr_memcache_add_server(b->memcache, server);
484 if (apr_err != APR_SUCCESS)
486 b->err = svn_error_wrap_apr(apr_err,
487 _("Unknown error adding server to memcache"));
494 #else /* ! SVN_HAVE_MEMCACHE */
496 /* Stubs for no apr memcache library. */
498 struct svn_memcache_t {
499 void *unused; /* Let's not have a size-zero struct. */
503 svn_cache__create_memcache(svn_cache__t **cache_p,
504 svn_memcache_t *memcache,
505 svn_cache__serialize_func_t serialize_func,
506 svn_cache__deserialize_func_t deserialize_func,
511 return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL);
514 #endif /* SVN_HAVE_MEMCACHE */
516 /* Implements svn_config_enumerator2_t. Just used for the
517 entry-counting return value of svn_config_enumerate2. */
519 nop_enumerator(const char *name,
528 svn_cache__make_memcache_from_config(svn_memcache_t **memcache_p,
529 svn_config_t *config,
533 apr_pool_t *subpool = svn_pool_create(pool);
536 svn_config_enumerate2(config,
537 SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS,
538 nop_enumerator, NULL, subpool);
540 if (server_count == 0)
543 svn_pool_destroy(subpool);
547 if (server_count > APR_INT16_MAX)
548 return svn_error_create(SVN_ERR_TOO_MANY_MEMCACHED_SERVERS, NULL, NULL);
550 #ifdef SVN_HAVE_MEMCACHE
553 svn_memcache_t *memcache = apr_pcalloc(pool, sizeof(*memcache));
554 apr_status_t apr_err = apr_memcache_create(pool,
555 (apr_uint16_t)server_count,
558 if (apr_err != APR_SUCCESS)
559 return svn_error_wrap_apr(apr_err,
560 _("Unknown error creating apr_memcache_t"));
562 b.memcache = memcache->c;
563 b.memcache_pool = pool;
564 b.err = SVN_NO_ERROR;
565 svn_config_enumerate2(config,
566 SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS,
567 add_memcache_server, &b,
573 *memcache_p = memcache;
575 svn_pool_destroy(subpool);
578 #else /* ! SVN_HAVE_MEMCACHE */
580 return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL);
582 #endif /* SVN_HAVE_MEMCACHE */