1 /* fs.c --- creating, opening and closing filesystems
3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
20 * ====================================================================
27 #include <apr_general.h>
28 #include <apr_pools.h>
29 #include <apr_file_io.h>
32 #include "svn_pools.h"
36 #include "svn_delta.h"
37 #include "svn_version.h"
41 #include "revs-txns.h"
47 #include "svn_private_config.h"
49 #include "bdb/bdb-err.h"
50 #include "bdb/bdb_compat.h"
52 #include "bdb/nodes-table.h"
53 #include "bdb/rev-table.h"
54 #include "bdb/txn-table.h"
55 #include "bdb/copies-table.h"
56 #include "bdb/changes-table.h"
57 #include "bdb/reps-table.h"
58 #include "bdb/strings-table.h"
59 #include "bdb/uuids-table.h"
60 #include "bdb/locks-table.h"
61 #include "bdb/lock-tokens-table.h"
62 #include "bdb/node-origins-table.h"
63 #include "bdb/miscellaneous-table.h"
64 #include "bdb/checksum-reps-table.h"
66 #include "../libsvn_fs/fs-loader.h"
67 #include "private/svn_fs_util.h"
70 /* Checking for return values, and reporting errors. */
72 /* Check that we're using the right Berkeley DB version. */
73 /* FIXME: This check should be abstracted into the DB back-end layer. */
75 check_bdb_version(void)
77 int major, minor, patch;
79 db_version(&major, &minor, &patch);
81 /* First, check that we're using a reasonably correct of Berkeley DB. */
82 if ((major < SVN_FS_WANT_DB_MAJOR)
83 || (major == SVN_FS_WANT_DB_MAJOR && minor < SVN_FS_WANT_DB_MINOR)
84 || (major == SVN_FS_WANT_DB_MAJOR && minor == SVN_FS_WANT_DB_MINOR
85 && patch < SVN_FS_WANT_DB_PATCH))
86 return svn_error_createf(SVN_ERR_FS_GENERAL, 0,
87 _("Bad database version: got %d.%d.%d,"
88 " should be at least %d.%d.%d"),
92 SVN_FS_WANT_DB_PATCH);
94 /* Now, check that the version we're running against is the same as
95 the one we compiled with. */
96 if (major != DB_VERSION_MAJOR || minor != DB_VERSION_MINOR)
97 return svn_error_createf(SVN_ERR_FS_GENERAL, 0,
98 _("Bad database version:"
99 " compiled with %d.%d.%d,"
100 " running against %d.%d.%d"),
104 major, minor, patch);
110 /* Cleanup functions. */
112 /* Close a database in the filesystem FS.
113 DB_PTR is a pointer to the DB pointer in *FS to close.
114 NAME is the name of the database, for use in error messages. */
116 cleanup_fs_db(svn_fs_t *fs, DB **db_ptr, const char *name)
118 /* If the BDB environment is panicked, don't do anything, since
119 attempting to close the database will fail anyway. */
120 base_fs_data_t *bfd = fs->fsap_data;
121 if (*db_ptr && !svn_fs_bdb__get_panic(bfd->bdb))
124 char *msg = apr_psprintf(fs->pool, "closing '%s' database", name);
128 db_err = db->close(db, 0);
129 if (db_err == DB_RUNRECOVERY)
131 /* We can ignore DB_RUNRECOVERY errors from DB->close, but
132 must set the panic flag in the environment baton. The
133 error will be propagated appropriately from
134 svn_fs_bdb__close. */
135 svn_fs_bdb__set_panic(bfd->bdb);
139 #if SVN_BDB_HAS_DB_INCOMPLETE
140 /* We can ignore DB_INCOMPLETE on db->close and db->sync; it
141 * just means someone else was using the db at the same time
142 * we were. See the Berkeley documentation at:
143 * http://www.sleepycat.com/docs/ref/program/errorret.html#DB_INCOMPLETE
144 * http://www.sleepycat.com/docs/api_c/db_close.html
146 if (db_err == DB_INCOMPLETE)
148 #endif /* SVN_BDB_HAS_DB_INCOMPLETE */
150 SVN_ERR(BDB_WRAP(fs, msg, db_err));
156 /* Close whatever Berkeley DB resources are allocated to FS. */
158 cleanup_fs(svn_fs_t *fs)
160 base_fs_data_t *bfd = fs->fsap_data;
161 bdb_env_baton_t *bdb = (bfd ? bfd->bdb : NULL);
166 /* Close the databases. */
167 SVN_ERR(cleanup_fs_db(fs, &bfd->nodes, "nodes"));
168 SVN_ERR(cleanup_fs_db(fs, &bfd->revisions, "revisions"));
169 SVN_ERR(cleanup_fs_db(fs, &bfd->transactions, "transactions"));
170 SVN_ERR(cleanup_fs_db(fs, &bfd->copies, "copies"));
171 SVN_ERR(cleanup_fs_db(fs, &bfd->changes, "changes"));
172 SVN_ERR(cleanup_fs_db(fs, &bfd->representations, "representations"));
173 SVN_ERR(cleanup_fs_db(fs, &bfd->strings, "strings"));
174 SVN_ERR(cleanup_fs_db(fs, &bfd->uuids, "uuids"));
175 SVN_ERR(cleanup_fs_db(fs, &bfd->locks, "locks"));
176 SVN_ERR(cleanup_fs_db(fs, &bfd->lock_tokens, "lock-tokens"));
177 SVN_ERR(cleanup_fs_db(fs, &bfd->node_origins, "node-origins"));
178 SVN_ERR(cleanup_fs_db(fs, &bfd->checksum_reps, "checksum-reps"));
179 SVN_ERR(cleanup_fs_db(fs, &bfd->miscellaneous, "miscellaneous"));
181 /* Finally, close the environment. */
184 svn_error_t *err = svn_fs_bdb__close(bdb);
186 return svn_error_createf
188 _("Berkeley DB error for filesystem '%s'"
189 " while closing environment:\n"),
195 #if 0 /* Set to 1 for instrumenting. */
196 static void print_fs_stats(svn_fs_t *fs)
198 base_fs_data_t *bfd = fs->fsap_data;
203 /* Print transaction statistics for this DB env. */
204 if ((db_err = bfd->bdb->env->txn_stat(bfd->bdb->env, &t, 0)) != 0)
205 fprintf(stderr, "Error running bfd->bdb->env->txn_stat(): %s",
206 db_strerror(db_err));
209 printf("*** DB transaction stats, right before closing env:\n");
210 printf(" Number of transactions currently active: %d\n",
212 printf(" Max number of active transactions at any one time: %d\n",
214 printf(" Number of transactions that have begun: %d\n",
216 printf(" Number of transactions that have aborted: %d\n",
218 printf(" Number of transactions that have committed: %d\n",
220 printf(" Number of times a thread was forced to wait: %d\n",
222 printf(" Number of times a thread didn't need to wait: %d\n",
223 t->st_region_nowait);
224 printf("*** End DB transaction stats.\n\n");
227 /* Print transaction statistics for this DB env. */
228 if ((db_err = bfd->bdb->env->lock_stat(bfd->bdb->env, &l, 0)) != 0)
229 fprintf(stderr, "Error running bfd->bdb->env->lock_stat(): %s",
230 db_strerror(db_err));
233 printf("*** DB lock stats, right before closing env:\n");
234 printf(" The number of current locks: %d\n",
236 printf(" Max number of locks at any one time: %d\n",
238 printf(" Number of current lockers: %d\n",
240 printf(" Max number of lockers at any one time: %d\n",
242 printf(" Number of current objects: %d\n",
244 printf(" Max number of objects at any one time: %d\n",
246 printf(" Total number of locks requested: %d\n",
248 printf(" Total number of locks released: %d\n",
250 printf(" Total number of lock reqs failed because "
251 "DB_LOCK_NOWAIT was set: %d\n", l->st_nnowaits);
252 printf(" Total number of locks not immediately available "
253 "due to conflicts: %d\n", l->st_nconflicts);
254 printf(" Number of deadlocks detected: %d\n", l->st_ndeadlocks);
255 printf(" Number of times a thread waited before "
256 "obtaining the region lock: %d\n", l->st_region_wait);
257 printf(" Number of times a thread didn't have to wait: %d\n",
258 l->st_region_nowait);
259 printf("*** End DB lock stats.\n\n");
264 # define print_fs_stats(fs)
267 /* An APR pool cleanup function for a filesystem. DATA must be a
268 pointer to the filesystem to clean up.
270 When the filesystem object's pool is freed, we want the resources
271 held by Berkeley DB to go away, just like everything else. So we
272 register this cleanup function with the filesystem's pool, and let
273 it take care of closing the databases, the environment, and any
274 other DB objects we might be using. APR calls this function before
275 actually freeing the pool's memory.
277 It's a pity that we can't return an svn_error_t object from an APR
278 cleanup function. For now, we return the rather generic
279 SVN_ERR_FS_CLEANUP, and pass the real svn_error_t to the registered
283 cleanup_fs_apr(void *data)
290 err = cleanup_fs(fs);
294 /* Darn. An error during cleanup. Call the warning handler to
295 try and do something "right" with this error. Note that
296 the default will simply abort(). */
297 (*fs->warning)(fs->warning_baton, err);
299 svn_error_clear(err);
301 return SVN_ERR_FS_CLEANUP;
306 base_bdb_set_errcall(svn_fs_t *fs,
307 void (*db_errcall_fcn)(const char *errpfx, char *msg))
309 base_fs_data_t *bfd = fs->fsap_data;
311 SVN_ERR(svn_fs__check_fs(fs, TRUE));
312 bfd->bdb->error_info->user_callback = db_errcall_fcn;
318 /* Write the DB_CONFIG file. */
320 bdb_write_config(svn_fs_t *fs)
322 const char *dbconfig_file_name =
323 svn_dirent_join(fs->path, BDB_CONFIG_FILE, fs->pool);
324 apr_file_t *dbconfig_file = NULL;
327 static const char dbconfig_contents[] =
328 "# This is the configuration file for the Berkeley DB environment\n"
329 "# used by your Subversion repository.\n"
330 "# You must run 'svnadmin recover' whenever you modify this file,\n"
331 "# for your changes to take effect.\n"
333 "### Lock subsystem\n"
335 "# Make sure you read the documentation at:\n"
337 "# http://docs.oracle.com/cd/E17076_02/html/programmer_reference/lock_max.html\n"
339 "# before tweaking these values.\n"
341 "set_lk_max_locks 2000\n"
342 "set_lk_max_lockers 2000\n"
343 "set_lk_max_objects 2000\n"
345 "### Log file subsystem\n"
347 "# Make sure you read the documentation at:\n"
349 "# http://docs.oracle.com/cd/E17076_02/html/api_reference/C/envset_lg_bsize.html\n"
350 "# http://docs.oracle.com/cd/E17076_02/html/api_reference/C/envset_lg_max.html\n"
351 "# http://docs.oracle.com/cd/E17076_02/html/programmer_reference/log_limits.html\n"
353 "# Increase the size of the in-memory log buffer from the default\n"
354 "# of 32 Kbytes to 256 Kbytes. Decrease the log file size from\n"
355 "# 10 Mbytes to 1 Mbyte. This will help reduce the amount of disk\n"
356 "# space required for hot backups. The size of the log file must be\n"
357 "# at least four times the size of the in-memory log buffer.\n"
359 "# Note: Decreasing the in-memory buffer size below 256 Kbytes will hurt\n"
360 "# hurt commit performance. For details, see:\n"
362 "# http://svn.haxx.se/dev/archive-2002-02/0141.shtml\n"
364 "set_lg_bsize 262144\n"
365 "set_lg_max 1048576\n"
367 "# If you see \"log region out of memory\" errors, bump lg_regionmax.\n"
368 "# For more information, see:\n"
370 "# http://docs.oracle.com/cd/E17076_02/html/programmer_reference/log_config.html\n"
371 "# http://svn.haxx.se/users/archive-2004-10/1000.shtml\n"
373 "set_lg_regionmax 131072\n"
375 /* ### Configure this with "svnadmin create --bdb-cache-size" */
376 "# The default cache size in BDB is only 256k. As explained in\n"
377 "# http://svn.haxx.se/dev/archive-2004-12/0368.shtml, this is too\n"
378 "# small for most applications. Bump this number if \"db_stat -m\"\n"
379 "# shows too many cache misses.\n"
381 "set_cachesize 0 1048576 1\n";
383 /* Run-time configurable options.
384 Each option set consists of a minimum required BDB version, a
385 config hash key, a header, an inactive form and an active
386 form. We always write the header; then, depending on the
387 run-time configuration and the BDB version we're compiling
388 against, we write either the active or inactive form of the
394 const char *config_key;
396 const char *inactive;
398 } dbconfig_options[] = {
399 /* Controlled by "svnadmin create --bdb-txn-nosync" */
400 { 4, 0, SVN_FS_CONFIG_BDB_TXN_NOSYNC,
403 "# Disable fsync of log files on transaction commit. Read the\n"
404 "# documentation about DB_TXN_NOSYNC at:\n"
406 "# http://docs.oracle.com/cd/E17076_02/html/programmer_reference/log_config.html\n"
408 "# [requires Berkeley DB 4.0]\n"
411 "#set_flags DB_TXN_NOSYNC\n",
413 "set_flags DB_TXN_NOSYNC\n" },
414 /* Controlled by "svnadmin create --bdb-log-keep" */
415 { 4, 2, SVN_FS_CONFIG_BDB_LOG_AUTOREMOVE,
418 "# Enable automatic removal of unused transaction log files.\n"
419 "# Read the documentation about DB_LOG_AUTOREMOVE at:\n"
421 "# http://docs.oracle.com/cd/E17076_02/html/programmer_reference/log_config.html\n"
423 "# [requires Berkeley DB 4.2]\n"
426 "#set_flags DB_LOG_AUTOREMOVE\n",
428 "set_flags DB_LOG_AUTOREMOVE\n" },
430 static const int dbconfig_options_length =
431 sizeof(dbconfig_options)/sizeof(*dbconfig_options);
434 SVN_ERR(svn_io_file_open(&dbconfig_file, dbconfig_file_name,
435 APR_WRITE | APR_CREATE, APR_OS_DEFAULT,
438 SVN_ERR(svn_io_file_write_full(dbconfig_file, dbconfig_contents,
439 sizeof(dbconfig_contents) - 1, NULL,
442 /* Write the variable DB_CONFIG flags. */
443 for (i = 0; i < dbconfig_options_length; ++i)
450 value = svn_hash_gets(fs->config, dbconfig_options[i].config_key);
453 SVN_ERR(svn_io_file_write_full(dbconfig_file,
454 dbconfig_options[i].header,
455 strlen(dbconfig_options[i].header),
458 if (((DB_VERSION_MAJOR == dbconfig_options[i].bdb_major
459 && DB_VERSION_MINOR >= dbconfig_options[i].bdb_minor)
460 || DB_VERSION_MAJOR > dbconfig_options[i].bdb_major)
461 && value != NULL && strcmp(value, "0") != 0)
462 choice = dbconfig_options[i].active;
464 choice = dbconfig_options[i].inactive;
466 SVN_ERR(svn_io_file_write_full(dbconfig_file, choice, strlen(choice),
470 return svn_io_file_close(dbconfig_file, fs->pool);
474 base_bdb_info_format(int *fs_format,
475 svn_version_t **supports_version,
477 apr_pool_t *result_pool,
478 apr_pool_t *scratch_pool)
480 base_fs_data_t *bfd = fs->fsap_data;
482 *fs_format = bfd->format;
483 *supports_version = apr_palloc(result_pool, sizeof(svn_version_t));
485 (*supports_version)->major = SVN_VER_MAJOR;
486 (*supports_version)->minor = 0;
487 (*supports_version)->patch = 0;
488 (*supports_version)->tag = "";
495 (*supports_version)->minor = 4;
498 (*supports_version)->minor = 5;
501 (*supports_version)->minor = 6;
504 # if SVN_FS_BASE__FORMAT_NUMBER != 4
505 # error "Need to add a 'case' statement here"
514 base_bdb_info_config_files(apr_array_header_t **files,
516 apr_pool_t *result_pool,
517 apr_pool_t *scratch_pool)
519 *files = apr_array_make(result_pool, 1, sizeof(const char *));
520 APR_ARRAY_PUSH(*files, const char *) = svn_dirent_join(fs->path,
527 base_bdb_verify_root(svn_fs_root_t *root,
528 apr_pool_t *scratch_pool)
530 /* Verifying is currently a no op for BDB. */
535 base_bdb_freeze(svn_fs_t *fs,
536 svn_fs_freeze_func_t freeze_func,
540 SVN__NOT_IMPLEMENTED();
544 /* Creating a new filesystem */
546 static fs_vtable_t fs_vtable = {
547 svn_fs_base__youngest_rev,
548 svn_fs_base__revision_prop,
549 svn_fs_base__revision_proplist,
550 svn_fs_base__change_rev_prop,
551 svn_fs_base__set_uuid,
552 svn_fs_base__revision_root,
553 svn_fs_base__begin_txn,
554 svn_fs_base__open_txn,
555 svn_fs_base__purge_txn,
556 svn_fs_base__list_transactions,
557 svn_fs_base__deltify,
559 svn_fs_base__generate_lock_token,
561 svn_fs_base__get_lock,
562 svn_fs_base__get_locks,
563 base_bdb_info_format,
564 base_bdb_info_config_files,
565 NULL /* info_fsap */,
566 base_bdb_verify_root,
568 base_bdb_set_errcall,
571 /* Where the format number is stored. */
572 #define FORMAT_FILE "format"
574 /* Depending on CREATE, create or open the environment and databases
575 for filesystem FS in PATH. Use POOL for temporary allocations. */
577 open_databases(svn_fs_t *fs,
578 svn_boolean_t create,
585 SVN_ERR(svn_fs__check_fs(fs, FALSE));
587 bfd = apr_pcalloc(fs->pool, sizeof(*bfd));
588 fs->vtable = &fs_vtable;
591 /* Initialize the fs's path. */
592 fs->path = apr_pstrdup(fs->pool, path);
595 SVN_ERR(bdb_write_config(fs));
597 /* Create the Berkeley DB environment. */
599 svn_error_t *err = svn_fs_bdb__open(&(bfd->bdb), path,
600 SVN_BDB_STANDARD_ENV_FLAGS,
605 return svn_error_createf
607 _("Berkeley DB error for filesystem '%s'"
608 " while creating environment:\n"),
611 return svn_error_createf
613 _("Berkeley DB error for filesystem '%s'"
614 " while opening environment:\n"),
619 /* We must register the FS cleanup function *after* opening the
620 environment, so that it's run before the environment baton
622 apr_pool_cleanup_register(fs->pool, fs, cleanup_fs_apr,
623 apr_pool_cleanup_null);
626 /* Create the databases in the environment. */
627 SVN_ERR(BDB_WRAP(fs, (create
628 ? N_("creating 'nodes' table")
629 : N_("opening 'nodes' table")),
630 svn_fs_bdb__open_nodes_table(&bfd->nodes,
633 SVN_ERR(BDB_WRAP(fs, (create
634 ? N_("creating 'revisions' table")
635 : N_("opening 'revisions' table")),
636 svn_fs_bdb__open_revisions_table(&bfd->revisions,
639 SVN_ERR(BDB_WRAP(fs, (create
640 ? N_("creating 'transactions' table")
641 : N_("opening 'transactions' table")),
642 svn_fs_bdb__open_transactions_table(&bfd->transactions,
645 SVN_ERR(BDB_WRAP(fs, (create
646 ? N_("creating 'copies' table")
647 : N_("opening 'copies' table")),
648 svn_fs_bdb__open_copies_table(&bfd->copies,
651 SVN_ERR(BDB_WRAP(fs, (create
652 ? N_("creating 'changes' table")
653 : N_("opening 'changes' table")),
654 svn_fs_bdb__open_changes_table(&bfd->changes,
657 SVN_ERR(BDB_WRAP(fs, (create
658 ? N_("creating 'representations' table")
659 : N_("opening 'representations' table")),
660 svn_fs_bdb__open_reps_table(&bfd->representations,
663 SVN_ERR(BDB_WRAP(fs, (create
664 ? N_("creating 'strings' table")
665 : N_("opening 'strings' table")),
666 svn_fs_bdb__open_strings_table(&bfd->strings,
669 SVN_ERR(BDB_WRAP(fs, (create
670 ? N_("creating 'uuids' table")
671 : N_("opening 'uuids' table")),
672 svn_fs_bdb__open_uuids_table(&bfd->uuids,
675 SVN_ERR(BDB_WRAP(fs, (create
676 ? N_("creating 'locks' table")
677 : N_("opening 'locks' table")),
678 svn_fs_bdb__open_locks_table(&bfd->locks,
681 SVN_ERR(BDB_WRAP(fs, (create
682 ? N_("creating 'lock-tokens' table")
683 : N_("opening 'lock-tokens' table")),
684 svn_fs_bdb__open_lock_tokens_table(&bfd->lock_tokens,
688 if (format >= SVN_FS_BASE__MIN_NODE_ORIGINS_FORMAT)
690 SVN_ERR(BDB_WRAP(fs, (create
691 ? N_("creating 'node-origins' table")
692 : N_("opening 'node-origins' table")),
693 svn_fs_bdb__open_node_origins_table(&bfd->node_origins,
698 if (format >= SVN_FS_BASE__MIN_MISCELLANY_FORMAT)
700 SVN_ERR(BDB_WRAP(fs, (create
701 ? N_("creating 'miscellaneous' table")
702 : N_("opening 'miscellaneous' table")),
703 svn_fs_bdb__open_miscellaneous_table(&bfd->miscellaneous,
708 if (format >= SVN_FS_BASE__MIN_REP_SHARING_FORMAT)
710 SVN_ERR(BDB_WRAP(fs, (create
711 ? N_("creating 'checksum-reps' table")
712 : N_("opening 'checksum-reps' table")),
713 svn_fs_bdb__open_checksum_reps_table(&bfd->checksum_reps,
722 /* Called by functions that initialize an svn_fs_t struct, after that
723 initialization is done, to populate svn_fs_t->uuid. */
725 populate_opened_fs(svn_fs_t *fs, apr_pool_t *scratch_pool)
727 SVN_ERR(svn_fs_base__populate_uuid(fs, scratch_pool));
732 base_create(svn_fs_t *fs,
734 svn_mutex__t *common_pool_lock,
736 apr_pool_t *common_pool)
738 int format = SVN_FS_BASE__FORMAT_NUMBER;
739 svn_error_t *svn_err;
741 /* See if compatibility with older versions was explicitly requested. */
744 svn_version_t *compatible_version;
745 SVN_ERR(svn_fs__compatible_version(&compatible_version, fs->config,
748 /* select format number */
749 switch(compatible_version->minor)
763 default:format = SVN_FS_BASE__FORMAT_NUMBER;
767 /* Create the environment and databases. */
768 svn_err = open_databases(fs, TRUE, format, path, pool);
769 if (svn_err) goto error;
771 /* Initialize the DAG subsystem. */
772 svn_err = svn_fs_base__dag_init_fs(fs);
773 if (svn_err) goto error;
775 /* This filesystem is ready. Stamp it with a format number. */
776 svn_err = svn_io_write_version_file(
777 svn_dirent_join(fs->path, FORMAT_FILE, pool), format, pool);
778 if (svn_err) goto error;
780 ((base_fs_data_t *) fs->fsap_data)->format = format;
782 SVN_ERR(populate_opened_fs(fs, pool));
783 return SVN_NO_ERROR;;
786 return svn_error_compose_create(svn_err,
787 svn_error_trace(cleanup_fs(fs)));
791 /* Gaining access to an existing Berkeley DB-based filesystem. */
794 svn_fs_base__test_required_feature_format(svn_fs_t *fs,
798 base_fs_data_t *bfd = fs->fsap_data;
799 if (bfd->format < requires)
800 return svn_error_createf
801 (SVN_ERR_UNSUPPORTED_FEATURE, NULL,
802 _("The '%s' feature requires version %d of the filesystem schema; "
803 "filesystem '%s' uses only version %d"),
804 feature, requires, fs->path, bfd->format);
808 /* Return the error SVN_ERR_FS_UNSUPPORTED_FORMAT if FS's format
809 number is not the same as the format number supported by this
812 check_format(int format)
814 /* We currently support any format less than the compiled format number
816 if (format <= SVN_FS_BASE__FORMAT_NUMBER)
819 return svn_error_createf(
820 SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL,
821 _("Expected FS format '%d'; found format '%d'"),
822 SVN_FS_BASE__FORMAT_NUMBER, format);
826 base_open(svn_fs_t *fs,
828 svn_mutex__t *common_pool_lock,
830 apr_pool_t *common_pool)
833 svn_error_t *svn_err;
834 svn_boolean_t write_format_file = FALSE;
836 /* Read the FS format number. */
837 svn_err = svn_io_read_version_file(&format,
838 svn_dirent_join(path, FORMAT_FILE, pool),
840 if (svn_err && APR_STATUS_IS_ENOENT(svn_err->apr_err))
842 /* Pre-1.2 filesystems did not have a format file (you could say
843 they were format "0"), so they get upgraded on the fly.
844 However, we stopped "upgrading on the fly" in 1.5, so older
845 filesystems should only be bumped to 1.3, which is format "1". */
846 svn_error_clear(svn_err);
847 svn_err = SVN_NO_ERROR;
849 write_format_file = TRUE;
854 /* Create the environment and databases. */
855 svn_err = open_databases(fs, FALSE, format, path, pool);
856 if (svn_err) goto error;
858 ((base_fs_data_t *) fs->fsap_data)->format = format;
859 SVN_ERR(check_format(format));
861 /* If we lack a format file, write one. */
862 if (write_format_file)
864 svn_err = svn_io_write_version_file(svn_dirent_join(path, FORMAT_FILE,
867 if (svn_err) goto error;
870 SVN_ERR(populate_opened_fs(fs, pool));
874 return svn_error_compose_create(svn_err,
875 svn_error_trace(cleanup_fs(fs)));
879 /* Running recovery on a Berkeley DB-based filesystem. */
882 /* Recover a database at PATH. Perform catastrophic recovery if FATAL
883 is TRUE. Use POOL for temporary allocation. */
885 bdb_recover(const char *path, svn_boolean_t fatal, apr_pool_t *pool)
887 bdb_env_baton_t *bdb;
889 /* Here's the comment copied from db_recover.c:
891 Initialize the environment -- we don't actually do anything
892 else, that all that's needed to run recovery.
894 Note that we specify a private environment, as we're about to
895 create a region, and we don't want to leave it around. If we
896 leave the region around, the application that should create it
897 will simply join it instead, and will then be running with
898 incorrectly sized (and probably terribly small) caches. */
900 /* Note that since we're using a private environment, we shoudl
901 /not/ initialize locking. We want the environment files to go
904 SVN_ERR(svn_fs_bdb__open(&bdb, path,
905 ((fatal ? DB_RECOVER_FATAL : DB_RECOVER)
906 | SVN_BDB_PRIVATE_ENV_FLAGS),
908 return svn_fs_bdb__close(bdb);
912 base_open_for_recovery(svn_fs_t *fs,
914 svn_mutex__t *common_pool_lock,
916 apr_pool_t *common_pool)
918 /* Just stash the path in the fs pointer - it's all we really need. */
919 fs->path = apr_pstrdup(fs->pool, path);
925 base_upgrade(svn_fs_t *fs,
927 svn_fs_upgrade_notify_t notify_func,
929 svn_cancel_func_t cancel_func,
931 svn_mutex__t *common_pool_lock,
933 apr_pool_t *common_pool)
935 const char *version_file_path;
936 int old_format_number;
939 version_file_path = svn_dirent_join(path, FORMAT_FILE, pool);
941 /* Read the old number so we've got it on hand later on. */
942 err = svn_io_read_version_file(&old_format_number, version_file_path, pool);
943 if (err && APR_STATUS_IS_ENOENT(err->apr_err))
945 /* Pre-1.2 filesystems do not have a 'format' file. */
946 old_format_number = 0;
947 svn_error_clear(err);
952 /* Bump the format file's stored version number. */
953 SVN_ERR(svn_io_write_version_file(version_file_path,
954 SVN_FS_BASE__FORMAT_NUMBER, pool));
956 SVN_ERR(notify_func(notify_baton, SVN_FS_BASE__FORMAT_NUMBER,
957 svn_fs_upgrade_format_bumped, pool));
959 /* Check and see if we need to record the "bump" revision. */
960 if (old_format_number < SVN_FS_BASE__MIN_FORWARD_DELTAS_FORMAT)
962 apr_pool_t *subpool = svn_pool_create(pool);
963 svn_revnum_t youngest_rev;
966 /* Open the filesystem in a subpool (so we can control its
967 closure) and do our fiddling.
969 NOTE: By using base_open() here instead of open_databases(),
970 we will end up re-reading the format file that we just wrote.
971 But it's better to use the existing encapsulation of "opening
972 the filesystem" rather than duplicating (or worse, partially
973 duplicating) that logic here. */
974 SVN_ERR(base_open(fs, path, common_pool_lock, subpool, common_pool));
976 /* Fetch the youngest rev, and record it */
977 SVN_ERR(svn_fs_base__youngest_rev(&youngest_rev, fs, subpool));
978 value = apr_psprintf(subpool, "%ld", youngest_rev);
979 SVN_ERR(svn_fs_base__miscellaneous_set
980 (fs, SVN_FS_BASE__MISC_FORWARD_DELTA_UPGRADE,
982 svn_pool_destroy(subpool);
989 base_verify(svn_fs_t *fs, const char *path,
992 svn_fs_progress_notify_func_t notify_func,
994 svn_cancel_func_t cancel_func,
996 svn_mutex__t *common_pool_lock,
998 apr_pool_t *common_pool)
1000 /* Verifying is currently a no op for BDB. */
1001 return SVN_NO_ERROR;
1004 static svn_error_t *
1005 base_bdb_recover(svn_fs_t *fs,
1006 svn_cancel_func_t cancel_func, void *cancel_baton,
1009 /* The fs pointer is a fake created in base_open_for_recovery above.
1010 We only care about the path. */
1011 return bdb_recover(fs->path, FALSE, pool);
1014 static svn_error_t *
1015 base_bdb_pack(svn_fs_t *fs,
1017 svn_fs_pack_notify_t notify_func,
1019 svn_cancel_func_t cancel,
1021 svn_mutex__t *common_pool_lock,
1023 apr_pool_t *common_pool)
1025 /* Packing is currently a no op for BDB. */
1026 return SVN_NO_ERROR;
1031 /* Running the 'archive' command on a Berkeley DB-based filesystem. */
1034 static svn_error_t *
1035 base_bdb_logfiles(apr_array_header_t **logfiles,
1037 svn_boolean_t only_unused,
1040 bdb_env_baton_t *bdb;
1043 u_int32_t flags = only_unused ? 0 : DB_ARCH_LOG;
1045 *logfiles = apr_array_make(pool, 4, sizeof(const char *));
1047 SVN_ERR(svn_fs_bdb__open(&bdb, path,
1048 SVN_BDB_STANDARD_ENV_FLAGS,
1050 SVN_BDB_ERR(bdb, bdb->env->log_archive(bdb->env, &filelist, flags));
1052 if (filelist == NULL)
1053 return svn_fs_bdb__close(bdb);
1055 for (filename = filelist; *filename != NULL; ++filename)
1057 APR_ARRAY_PUSH(*logfiles, const char *) = apr_pstrdup(pool, *filename);
1062 return svn_fs_bdb__close(bdb);
1067 /* Copying a live Berkeley DB-base filesystem. */
1070 * Delete all unused log files from DBD enviroment at @a live_path that exist
1071 * in @a backup_path.
1073 static svn_error_t *
1074 svn_fs_base__clean_logs(const char *live_path,
1075 const char *backup_path,
1078 apr_array_header_t *logfiles;
1080 SVN_ERR(base_bdb_logfiles(&logfiles,
1082 TRUE, /* Only unused logs */
1085 { /* Process unused logs from live area */
1087 apr_pool_t *subpool = svn_pool_create(pool);
1089 /* Process log files. */
1090 for (idx = 0; idx < logfiles->nelts; idx++)
1092 const char *log_file = APR_ARRAY_IDX(logfiles, idx, const char *);
1093 const char *live_log_path;
1094 const char *backup_log_path;
1096 svn_pool_clear(subpool);
1097 live_log_path = svn_dirent_join(live_path, log_file, subpool);
1098 backup_log_path = svn_dirent_join(backup_path, log_file, subpool);
1100 { /* Compare files. No point in using MD5 and wasting CPU cycles as we
1101 got full copies of both logs */
1103 svn_boolean_t files_match = FALSE;
1104 svn_node_kind_t kind;
1106 /* Check to see if there is a corresponding log file in the backup
1108 SVN_ERR(svn_io_check_path(backup_log_path, &kind, pool));
1110 /* If the copy of the log exists, compare them */
1111 if (kind == svn_node_file)
1112 SVN_ERR(svn_io_files_contents_same_p(&files_match,
1117 /* If log files do not match, go to the next log file. */
1122 SVN_ERR(svn_io_remove_file2(live_log_path, FALSE, subpool));
1125 svn_pool_destroy(subpool);
1128 return SVN_NO_ERROR;
1132 /* DB_ENV->get_flags() and DB->get_pagesize() don't exist prior to
1134 #if SVN_BDB_VERSION_AT_LEAST(4, 2)
1136 /* Open the BDB environment at PATH and compare its configuration
1137 flags with FLAGS. If every flag in FLAGS is set in the
1138 environment, then set *MATCH to true. Else set *MATCH to false. */
1139 static svn_error_t *
1140 check_env_flags(svn_boolean_t *match,
1145 bdb_env_baton_t *bdb;
1146 #if SVN_BDB_VERSION_AT_LEAST(4, 7)
1152 SVN_ERR(svn_fs_bdb__open(&bdb, path,
1153 SVN_BDB_STANDARD_ENV_FLAGS,
1155 #if SVN_BDB_VERSION_AT_LEAST(4, 7)
1156 SVN_BDB_ERR(bdb, bdb->env->log_get_config(bdb->env, flags, &flag_state));
1158 SVN_BDB_ERR(bdb, bdb->env->get_flags(bdb->env, &envflags));
1161 SVN_ERR(svn_fs_bdb__close(bdb));
1163 #if SVN_BDB_VERSION_AT_LEAST(4, 7)
1164 if (flag_state == 0)
1166 if (flags & envflags)
1172 return SVN_NO_ERROR;
1176 /* Set *PAGESIZE to the size of pages used to hold items in the
1177 database environment located at PATH.
1179 static svn_error_t *
1180 get_db_pagesize(u_int32_t *pagesize,
1184 bdb_env_baton_t *bdb;
1187 SVN_ERR(svn_fs_bdb__open(&bdb, path,
1188 SVN_BDB_STANDARD_ENV_FLAGS,
1191 /* ### We're only asking for the pagesize on the 'nodes' table.
1192 Is this enough? We never call DB->set_pagesize() on any of
1193 our tables, so presumably BDB is using the same default
1194 pagesize for all our databases, right? */
1195 SVN_BDB_ERR(bdb, svn_fs_bdb__open_nodes_table(&nodes_table, bdb->env,
1197 SVN_BDB_ERR(bdb, nodes_table->get_pagesize(nodes_table, pagesize));
1198 SVN_BDB_ERR(bdb, nodes_table->close(nodes_table, 0));
1200 return svn_fs_bdb__close(bdb);
1202 #endif /* SVN_BDB_VERSION_AT_LEAST(4, 2) */
1205 /* Copy FILENAME from SRC_DIR to DST_DIR in byte increments of size
1206 CHUNKSIZE. The read/write buffer of size CHUNKSIZE will be
1207 allocated in POOL. If ALLOW_MISSING is set, we won't make a fuss
1208 if FILENAME isn't found in SRC_DIR; otherwise, we will. */
1209 static svn_error_t *
1210 copy_db_file_safely(const char *src_dir,
1211 const char *dst_dir,
1212 const char *filename,
1213 u_int32_t chunksize,
1214 svn_boolean_t allow_missing,
1217 apr_file_t *s = NULL, *d = NULL; /* init to null important for APR */
1218 const char *file_src_path = svn_dirent_join(src_dir, filename, pool);
1219 const char *file_dst_path = svn_dirent_join(dst_dir, filename, pool);
1223 /* Open source file. If it's missing and that's allowed, there's
1224 nothing more to do here. */
1225 err = svn_io_file_open(&s, file_src_path,
1226 (APR_READ | APR_LARGEFILE),
1227 APR_OS_DEFAULT, pool);
1228 if (err && APR_STATUS_IS_ENOENT(err->apr_err) && allow_missing)
1230 svn_error_clear(err);
1231 return SVN_NO_ERROR;
1235 /* Open destination file. */
1236 SVN_ERR(svn_io_file_open(&d, file_dst_path, (APR_WRITE | APR_CREATE |
1238 APR_OS_DEFAULT, pool));
1240 /* Allocate our read/write buffer. */
1241 buf = apr_palloc(pool, chunksize);
1243 /* Copy bytes till the cows come home. */
1246 apr_size_t bytes_this_time = chunksize;
1247 svn_error_t *read_err, *write_err;
1250 if ((read_err = svn_io_file_read(s, buf, &bytes_this_time, pool)))
1252 if (APR_STATUS_IS_EOF(read_err->apr_err))
1253 svn_error_clear(read_err);
1256 svn_error_clear(svn_io_file_close(s, pool));
1257 svn_error_clear(svn_io_file_close(d, pool));
1263 if ((write_err = svn_io_file_write_full(d, buf, bytes_this_time, NULL,
1266 svn_error_clear(svn_io_file_close(s, pool));
1267 svn_error_clear(svn_io_file_close(d, pool));
1271 /* read_err is either NULL, or a dangling pointer - but it is only a
1272 dangling pointer if it used to be an EOF error. */
1275 SVN_ERR(svn_io_file_close(s, pool));
1276 SVN_ERR(svn_io_file_close(d, pool));
1277 break; /* got EOF on read, all files closed, all done. */
1281 return SVN_NO_ERROR;
1287 static svn_error_t *
1288 base_hotcopy(svn_fs_t *src_fs,
1290 const char *src_path,
1291 const char *dest_path,
1292 svn_boolean_t clean_logs,
1293 svn_boolean_t incremental,
1294 svn_fs_hotcopy_notify_t notify_func,
1296 svn_cancel_func_t cancel_func,
1298 svn_mutex__t *common_pool_lock,
1300 apr_pool_t *common_pool)
1304 svn_boolean_t log_autoremove = FALSE;
1308 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
1309 _("BDB repositories do not support incremental "
1312 /* Check the FS format number to be certain that we know how to
1313 hotcopy this FS. Pre-1.2 filesystems did not have a format file (you
1314 could say they were format "0"), so we will error here. This is not
1315 optimal, but since this has been the case since 1.2.0, and no one has
1316 complained, it apparently isn't much of a concern. (We did not check
1317 the 'format' file in 1.2.x, but we did blindly try to copy 'locks',
1318 which would have errored just the same.) */
1319 SVN_ERR(svn_io_read_version_file(
1320 &format, svn_dirent_join(src_path, FORMAT_FILE, pool), pool));
1321 SVN_ERR(check_format(format));
1323 /* If using Berkeley DB 4.2 or later, note whether the DB_LOG_AUTO_REMOVE
1324 feature is on. If it is, we have a potential race condition:
1325 another process might delete a logfile while we're in the middle
1326 of copying all the logfiles. (This is not a huge deal; at worst,
1327 the hotcopy fails with a file-not-found error.) */
1328 #if SVN_BDB_VERSION_AT_LEAST(4, 2)
1329 err = check_env_flags(&log_autoremove,
1330 #if SVN_BDB_VERSION_AT_LEAST(4, 7)
1332 /* DB_LOG_AUTO_REMOVE was named DB_LOG_AUTOREMOVE before Berkeley DB 4.7. */
1340 /* Copy the DB_CONFIG file. */
1341 SVN_ERR(svn_io_dir_file_copy(src_path, dest_path, "DB_CONFIG", pool));
1343 /* In order to copy the database files safely and atomically, we
1344 must copy them in chunks which are multiples of the page-size
1345 used by BDB. See sleepycat docs for details, or svn issue #1818. */
1346 #if SVN_BDB_VERSION_AT_LEAST(4, 2)
1347 SVN_ERR(get_db_pagesize(&pagesize, src_path, pool));
1348 if (pagesize < SVN__STREAM_CHUNK_SIZE)
1350 /* use the largest multiple of BDB pagesize we can. */
1351 int multiple = SVN__STREAM_CHUNK_SIZE / pagesize;
1352 pagesize *= multiple;
1355 /* default to 128K chunks, which should be safe.
1356 BDB almost certainly uses a power-of-2 pagesize. */
1357 pagesize = (4096 * 32);
1360 /* Copy the databases. */
1361 SVN_ERR(copy_db_file_safely(src_path, dest_path,
1362 "nodes", pagesize, FALSE, pool));
1363 SVN_ERR(copy_db_file_safely(src_path, dest_path,
1364 "transactions", pagesize, FALSE, pool));
1365 SVN_ERR(copy_db_file_safely(src_path, dest_path,
1366 "revisions", pagesize, FALSE, pool));
1367 SVN_ERR(copy_db_file_safely(src_path, dest_path,
1368 "copies", pagesize, FALSE, pool));
1369 SVN_ERR(copy_db_file_safely(src_path, dest_path,
1370 "changes", pagesize, FALSE, pool));
1371 SVN_ERR(copy_db_file_safely(src_path, dest_path,
1372 "representations", pagesize, FALSE, pool));
1373 SVN_ERR(copy_db_file_safely(src_path, dest_path,
1374 "strings", pagesize, FALSE, pool));
1375 SVN_ERR(copy_db_file_safely(src_path, dest_path,
1376 "uuids", pagesize, TRUE, pool));
1377 SVN_ERR(copy_db_file_safely(src_path, dest_path,
1378 "locks", pagesize, TRUE, pool));
1379 SVN_ERR(copy_db_file_safely(src_path, dest_path,
1380 "lock-tokens", pagesize, TRUE, pool));
1381 SVN_ERR(copy_db_file_safely(src_path, dest_path,
1382 "node-origins", pagesize, TRUE, pool));
1383 SVN_ERR(copy_db_file_safely(src_path, dest_path,
1384 "checksum-reps", pagesize, TRUE, pool));
1385 SVN_ERR(copy_db_file_safely(src_path, dest_path,
1386 "miscellaneous", pagesize, TRUE, pool));
1389 apr_array_header_t *logfiles;
1391 apr_pool_t *subpool;
1393 SVN_ERR(base_bdb_logfiles(&logfiles,
1395 FALSE, /* All logs */
1398 /* Process log files. */
1399 subpool = svn_pool_create(pool);
1400 for (idx = 0; idx < logfiles->nelts; idx++)
1402 svn_pool_clear(subpool);
1403 err = svn_io_dir_file_copy(src_path, dest_path,
1404 APR_ARRAY_IDX(logfiles, idx,
1411 svn_error_quick_wrap
1413 _("Error copying logfile; the DB_LOG_AUTOREMOVE feature\n"
1414 "may be interfering with the hotcopy algorithm. If\n"
1415 "the problem persists, try deactivating this feature\n"
1418 return svn_error_trace(err);
1421 svn_pool_destroy(subpool);
1424 /* Since this is a copy we will have exclusive access to the repository. */
1425 err = bdb_recover(dest_path, TRUE, pool);
1430 svn_error_quick_wrap
1432 _("Error running catastrophic recovery on hotcopy; the\n"
1433 "DB_LOG_AUTOREMOVE feature may be interfering with the\n"
1434 "hotcopy algorithm. If the problem persists, try deactivating\n"
1435 "this feature in DB_CONFIG"));
1437 return svn_error_trace(err);
1440 /* Only now that the hotcopied filesystem is complete,
1441 stamp it with a format file. */
1442 SVN_ERR(svn_io_write_version_file(
1443 svn_dirent_join(dest_path, FORMAT_FILE, pool), format, pool));
1446 SVN_ERR(svn_fs_base__clean_logs(src_path, dest_path, pool));
1448 return SVN_NO_ERROR;
1453 /* Deleting a Berkeley DB-based filesystem. */
1456 static svn_error_t *
1457 base_delete_fs(const char *path,
1460 /* First, use the Berkeley DB library function to remove any shared
1462 SVN_ERR(svn_fs_bdb__remove(path, pool));
1464 /* Remove the environment directory. */
1465 return svn_io_remove_dir2(path, FALSE, NULL, NULL, pool);
1468 static const svn_version_t *
1475 base_get_description(void)
1477 return _("Module for working with a Berkeley DB repository.");
1480 static svn_error_t *
1481 base_set_svn_fs_open(svn_fs_t *fs,
1482 svn_error_t *(*svn_fs_open_)(svn_fs_t **,
1488 return SVN_NO_ERROR;
1492 /* Base FS library vtable, used by the FS loader library. */
1493 static fs_library_vtable_t library_vtable = {
1497 base_open_for_recovery,
1502 base_get_description,
1506 svn_fs_base__id_parse,
1507 base_set_svn_fs_open,
1508 NULL /* info_fsap_dup */
1512 svn_fs_base__init(const svn_version_t *loader_version,
1513 fs_library_vtable_t **vtable, apr_pool_t* common_pool)
1515 static const svn_version_checklist_t checklist[] =
1517 { "svn_subr", svn_subr_version },
1518 { "svn_delta", svn_delta_version },
1519 { "svn_fs_util", svn_fs_util__version },
1523 /* Simplified version check to make sure we can safely use the
1524 VTABLE parameter. The FS loader does a more exhaustive check. */
1525 if (loader_version->major != SVN_VER_MAJOR)
1526 return svn_error_createf(SVN_ERR_VERSION_MISMATCH, NULL,
1527 _("Unsupported FS loader version (%d) for bdb"),
1528 loader_version->major);
1529 SVN_ERR(svn_ver_check_list2(base_version(), checklist, svn_ver_equal));
1530 SVN_ERR(check_bdb_version());
1531 SVN_ERR(svn_fs_bdb__init(common_pool));
1533 *vtable = &library_vtable;
1534 return SVN_NO_ERROR;