2 * cache-memcache.c: memcached caching for Subversion
4 * ====================================================================
5 * Licensed to the Apache Software Foundation (ASF) under one
6 * or more contributor license agreements. See the NOTICE file
7 * distributed with this work for additional information
8 * regarding copyright ownership. The ASF licenses this file
9 * to you under the Apache License, Version 2.0 (the
10 * "License"); you may not use this file except in compliance
11 * with the License. You may obtain a copy of the License at
13 * http://www.apache.org/licenses/LICENSE-2.0
15 * Unless required by applicable law or agreed to in writing,
16 * software distributed under the License is distributed on an
17 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
18 * KIND, either express or implied. See the License for the
19 * specific language governing permissions and limitations
21 * ====================================================================
26 #include "svn_pools.h"
27 #include "svn_base64.h"
30 #include "svn_private_config.h"
31 #include "private/svn_cache.h"
32 #include "private/svn_dep_compat.h"
36 #ifdef SVN_HAVE_MEMCACHE
38 #include <apr_memcache.h>
40 /* A note on thread safety:
42 The apr_memcache_t object does its own mutex handling, and nothing
43 else in memcache_t is ever modified, so this implementation should
47 /* The (internal) cache object. */
48 typedef struct memcache_t {
49 /* The memcached server set we're using. */
50 apr_memcache_t *memcache;
52 /* A prefix used to differentiate our data from any other data in
53 * the memcached (URI-encoded). */
56 /* The size of the key: either a fixed number of bytes or
57 * APR_HASH_KEY_STRING. */
61 /* Used to marshal values in and out of the cache. */
62 svn_cache__serialize_func_t serialize_func;
63 svn_cache__deserialize_func_t deserialize_func;
66 /* The wrapper around apr_memcache_t. */
67 struct svn_memcache_t {
72 /* The memcached protocol says the maximum key length is 250. Let's
73 just say 249, to be safe. */
74 #define MAX_MEMCACHED_KEY_LEN 249
75 #define MEMCACHED_KEY_UNHASHED_LEN (MAX_MEMCACHED_KEY_LEN - \
76 2 * APR_MD5_DIGESTSIZE)
79 /* Set *MC_KEY to a memcache key for the given key KEY for CACHE, allocated
82 build_key(const char **mc_key,
87 const char *encoded_suffix;
89 apr_size_t long_key_len;
91 if (cache->klen == APR_HASH_KEY_STRING)
92 encoded_suffix = svn_path_uri_encode(raw_key, pool);
95 const svn_string_t *raw = svn_string_ncreate(raw_key, cache->klen, pool);
96 const svn_string_t *encoded = svn_base64_encode_string2(raw, FALSE,
98 encoded_suffix = encoded->data;
101 long_key = apr_pstrcat(pool, "SVN:", cache->prefix, ":", encoded_suffix,
103 long_key_len = strlen(long_key);
105 /* We don't want to have a key that's too big. If it was going to
106 be too big, we MD5 the entire string, then replace the last bit
107 with the checksum. Note that APR_MD5_DIGESTSIZE is for the pure
108 binary digest; we have to double that when we convert to hex.
110 Every key we use will either be at most
111 MEMCACHED_KEY_UNHASHED_LEN bytes long, or be exactly
112 MAX_MEMCACHED_KEY_LEN bytes long. */
113 if (long_key_len > MEMCACHED_KEY_UNHASHED_LEN)
115 svn_checksum_t *checksum;
116 SVN_ERR(svn_checksum(&checksum, svn_checksum_md5, long_key, long_key_len,
119 long_key = apr_pstrcat(pool,
120 apr_pstrmemdup(pool, long_key,
121 MEMCACHED_KEY_UNHASHED_LEN),
122 svn_checksum_to_cstring_display(checksum, pool),
130 /* Core functionality of our getter functions: fetch DATA from the memcached
131 * given by CACHE_VOID and identified by KEY. Indicate success in FOUND and
132 * use a tempoary sub-pool of POOL for allocations.
135 memcache_internal_get(char **data,
137 svn_boolean_t *found,
142 memcache_t *cache = cache_void;
143 apr_status_t apr_err;
153 subpool = svn_pool_create(pool);
154 SVN_ERR(build_key(&mc_key, cache, key, subpool));
156 apr_err = apr_memcache_getp(cache->memcache,
161 NULL /* ignore flags */);
162 if (apr_err == APR_NOTFOUND)
165 svn_pool_destroy(subpool);
168 else if (apr_err != APR_SUCCESS || !*data)
169 return svn_error_wrap_apr(apr_err,
170 _("Unknown memcached error while reading"));
174 svn_pool_destroy(subpool);
180 memcache_get(void **value_p,
181 svn_boolean_t *found,
184 apr_pool_t *result_pool)
186 memcache_t *cache = cache_void;
189 SVN_ERR(memcache_internal_get(&data,
196 /* If we found it, de-serialize it. */
199 if (cache->deserialize_func)
201 SVN_ERR((cache->deserialize_func)(value_p, data, data_len,
206 svn_stringbuf_t *value = svn_stringbuf_create_empty(result_pool);
208 value->blocksize = data_len;
209 value->len = data_len - 1; /* account for trailing NUL */
217 /* Core functionality of our setter functions: store LENGH bytes of DATA
218 * to be identified by KEY in the memcached given by CACHE_VOID. Use POOL
219 * for temporary allocations.
222 memcache_internal_set(void *cache_void,
226 apr_pool_t *scratch_pool)
228 memcache_t *cache = cache_void;
230 apr_status_t apr_err;
232 SVN_ERR(build_key(&mc_key, cache, key, scratch_pool));
233 apr_err = apr_memcache_set(cache->memcache, mc_key, (char *)data, len, 0, 0);
235 /* ### Maybe write failures should be ignored (but logged)? */
236 if (apr_err != APR_SUCCESS)
237 return svn_error_wrap_apr(apr_err,
238 _("Unknown memcached error while writing"));
245 memcache_set(void *cache_void,
248 apr_pool_t *scratch_pool)
250 memcache_t *cache = cache_void;
251 apr_pool_t *subpool = svn_pool_create(scratch_pool);
259 if (cache->serialize_func)
261 SVN_ERR((cache->serialize_func)(&data, &data_len, value, subpool));
265 svn_stringbuf_t *value_str = value;
266 data = value_str->data;
267 data_len = value_str->len + 1; /* copy trailing NUL */
270 err = memcache_internal_set(cache_void, key, data, data_len, subpool);
272 svn_pool_destroy(subpool);
277 memcache_get_partial(void **value_p,
278 svn_boolean_t *found,
281 svn_cache__partial_getter_func_t func,
283 apr_pool_t *result_pool)
285 svn_error_t *err = SVN_NO_ERROR;
289 SVN_ERR(memcache_internal_get(&data,
296 /* If we found it, de-serialize it. */
298 ? func(value_p, data, size, baton, result_pool)
304 memcache_set_partial(void *cache_void,
306 svn_cache__partial_setter_func_t func,
308 apr_pool_t *scratch_pool)
310 svn_error_t *err = SVN_NO_ERROR;
314 svn_boolean_t found = FALSE;
316 apr_pool_t *subpool = svn_pool_create(scratch_pool);
317 SVN_ERR(memcache_internal_get((char **)&data,
324 /* If we found it, modify it and write it back to cache */
327 SVN_ERR(func(&data, &size, baton, subpool));
328 err = memcache_internal_set(cache_void, key, data, size, subpool);
331 svn_pool_destroy(subpool);
337 memcache_iter(svn_boolean_t *completed,
339 svn_iter_apr_hash_cb_t user_cb,
341 apr_pool_t *scratch_pool)
343 return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
344 _("Can't iterate a memcached cache"));
348 memcache_is_cachable(void *unused, apr_size_t size)
350 (void)unused; /* silence gcc warning. */
352 /* The memcached cutoff seems to be a bit (header length?) under a megabyte.
353 * We round down a little to be safe.
355 return size < 1000000;
359 memcache_get_info(void *cache_void,
360 svn_cache__info_t *info,
362 apr_pool_t *result_pool)
364 memcache_t *cache = cache_void;
366 info->id = apr_pstrdup(result_pool, cache->prefix);
368 /* we don't have any memory allocation info */
371 info->total_size = 0;
373 info->used_entries = 0;
374 info->total_entries = 0;
379 static svn_cache__vtable_t memcache_vtable = {
383 memcache_is_cachable,
384 memcache_get_partial,
385 memcache_set_partial,
390 svn_cache__create_memcache(svn_cache__t **cache_p,
391 svn_memcache_t *memcache,
392 svn_cache__serialize_func_t serialize_func,
393 svn_cache__deserialize_func_t deserialize_func,
398 svn_cache__t *wrapper = apr_pcalloc(pool, sizeof(*wrapper));
399 memcache_t *cache = apr_pcalloc(pool, sizeof(*cache));
401 cache->serialize_func = serialize_func;
402 cache->deserialize_func = deserialize_func;
404 cache->prefix = svn_path_uri_encode(prefix, pool);
405 cache->memcache = memcache->c;
407 wrapper->vtable = &memcache_vtable;
408 wrapper->cache_internal = cache;
409 wrapper->error_handler = 0;
410 wrapper->error_baton = 0;
417 /*** Creating apr_memcache_t from svn_config_t. ***/
419 /* Baton for add_memcache_server. */
421 apr_memcache_t *memcache;
422 apr_pool_t *memcache_pool;
426 /* Implements svn_config_enumerator2_t. */
428 add_memcache_server(const char *name,
433 struct ams_baton *b = baton;
436 apr_status_t apr_err;
437 apr_memcache_server_t *server;
439 apr_err = apr_parse_addr_port(&host, &scope, &port,
441 if (apr_err != APR_SUCCESS)
443 b->err = svn_error_wrap_apr(apr_err,
444 _("Error parsing memcache server '%s'"),
451 b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL,
452 _("Scope not allowed in memcache server "
459 b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL,
460 _("Must specify host and port for memcache "
466 /* Note: the four numbers here are only relevant when an
467 apr_memcache_t is being shared by multiple threads. */
468 apr_err = apr_memcache_server_create(b->memcache_pool,
471 0, /* min connections */
472 5, /* soft max connections */
473 10, /* hard max connections */
474 /* time to live (in microseconds) */
475 apr_time_from_sec(50),
477 if (apr_err != APR_SUCCESS)
479 b->err = svn_error_wrap_apr(apr_err,
480 _("Unknown error creating memcache server"));
484 apr_err = apr_memcache_add_server(b->memcache, server);
485 if (apr_err != APR_SUCCESS)
487 b->err = svn_error_wrap_apr(apr_err,
488 _("Unknown error adding server to memcache"));
495 #else /* ! SVN_HAVE_MEMCACHE */
497 /* Stubs for no apr memcache library. */
499 struct svn_memcache_t {
500 void *unused; /* Let's not have a size-zero struct. */
504 svn_cache__create_memcache(svn_cache__t **cache_p,
505 svn_memcache_t *memcache,
506 svn_cache__serialize_func_t serialize_func,
507 svn_cache__deserialize_func_t deserialize_func,
512 return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL);
515 #endif /* SVN_HAVE_MEMCACHE */
517 /* Implements svn_config_enumerator2_t. Just used for the
518 entry-counting return value of svn_config_enumerate2. */
520 nop_enumerator(const char *name,
529 svn_cache__make_memcache_from_config(svn_memcache_t **memcache_p,
530 svn_config_t *config,
534 apr_pool_t *subpool = svn_pool_create(pool);
537 svn_config_enumerate2(config,
538 SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS,
539 nop_enumerator, NULL, subpool);
541 if (server_count == 0)
544 svn_pool_destroy(subpool);
548 if (server_count > APR_INT16_MAX)
549 return svn_error_create(SVN_ERR_TOO_MANY_MEMCACHED_SERVERS, NULL, NULL);
551 #ifdef SVN_HAVE_MEMCACHE
554 svn_memcache_t *memcache = apr_pcalloc(pool, sizeof(*memcache));
555 apr_status_t apr_err = apr_memcache_create(pool,
556 (apr_uint16_t)server_count,
559 if (apr_err != APR_SUCCESS)
560 return svn_error_wrap_apr(apr_err,
561 _("Unknown error creating apr_memcache_t"));
563 b.memcache = memcache->c;
564 b.memcache_pool = pool;
565 b.err = SVN_NO_ERROR;
566 svn_config_enumerate2(config,
567 SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS,
568 add_memcache_server, &b,
574 *memcache_p = memcache;
576 svn_pool_destroy(subpool);
579 #else /* ! SVN_HAVE_MEMCACHE */
581 return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL);
583 #endif /* SVN_HAVE_MEMCACHE */