1 /* hotcopy.c --- FS hotcopy functionality for FSFS
3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
20 * ====================================================================
22 #include "svn_pools.h"
24 #include "svn_dirent_uri.h"
31 #include "rep-cache.h"
33 #include "../libsvn_fs/fs-loader.h"
35 #include "svn_private_config.h"
37 /* Like svn_io_dir_file_copy(), but doesn't copy files that exist at
38 * the destination and do not differ in terms of kind, size, and mtime.
39 * Set *SKIPPED_P to FALSE only if the file was copied, do not change
40 * the value in *SKIPPED_P otherwise. SKIPPED_P may be NULL if not
43 hotcopy_io_dir_file_copy(svn_boolean_t *skipped_p,
47 apr_pool_t *scratch_pool)
49 const svn_io_dirent2_t *src_dirent;
50 const svn_io_dirent2_t *dst_dirent;
51 const char *src_target;
52 const char *dst_target;
54 /* Does the destination already exist? If not, we must copy it. */
55 dst_target = svn_dirent_join(dst_path, file, scratch_pool);
56 SVN_ERR(svn_io_stat_dirent2(&dst_dirent, dst_target, FALSE, TRUE,
57 scratch_pool, scratch_pool));
58 if (dst_dirent->kind != svn_node_none)
60 /* If the destination's stat information indicates that the file
61 * is equal to the source, don't bother copying the file again. */
62 src_target = svn_dirent_join(src_path, file, scratch_pool);
63 SVN_ERR(svn_io_stat_dirent2(&src_dirent, src_target, FALSE, FALSE,
64 scratch_pool, scratch_pool));
65 if (src_dirent->kind == dst_dirent->kind &&
66 src_dirent->special == dst_dirent->special &&
67 src_dirent->filesize == dst_dirent->filesize &&
68 src_dirent->mtime <= dst_dirent->mtime)
75 return svn_error_trace(svn_io_dir_file_copy(src_path, dst_path, file,
79 /* Set *NAME_P to the UTF-8 representation of directory entry NAME.
80 * NAME is in the internal encoding used by APR; PARENT is in
81 * UTF-8 and in internal (not local) style.
83 * Use PARENT only for generating an error string if the conversion
84 * fails because NAME could not be represented in UTF-8. In that
85 * case, return a two-level error in which the outer error's message
86 * mentions PARENT, but the inner error's message does not mention
87 * NAME (except possibly in hex) since NAME may not be printable.
88 * Such a compound error at least allows the user to go looking in the
89 * right directory for the problem.
91 * If there is any other error, just return that error directly.
93 * If there is any error, the effect on *NAME_P is undefined.
95 * *NAME_P and NAME may refer to the same storage.
98 entry_name_to_utf8(const char **name_p,
103 svn_error_t *err = svn_path_cstring_to_utf8(name_p, name, pool);
104 if (err && err->apr_err == APR_EINVAL)
106 return svn_error_createf(err->apr_err, err,
107 _("Error converting entry "
108 "in directory '%s' to UTF-8"),
109 svn_dirent_local_style(parent, pool));
114 /* Like svn_io_copy_dir_recursively() but doesn't copy regular files that
115 * exist in the destination and do not differ from the source in terms of
116 * kind, size, and mtime. Set *SKIPPED_P to FALSE only if at least one
117 * file was copied, do not change the value in *SKIPPED_P otherwise.
118 * SKIPPED_P may be NULL if not required. */
120 hotcopy_io_copy_dir_recursively(svn_boolean_t *skipped_p,
122 const char *dst_parent,
123 const char *dst_basename,
124 svn_boolean_t copy_perms,
125 svn_cancel_func_t cancel_func,
129 svn_node_kind_t kind;
131 const char *dst_path;
133 apr_finfo_t this_entry;
134 apr_int32_t flags = APR_FINFO_TYPE | APR_FINFO_NAME;
136 /* Make a subpool for recursion */
137 apr_pool_t *subpool = svn_pool_create(pool);
139 /* The 'dst_path' is simply dst_parent/dst_basename */
140 dst_path = svn_dirent_join(dst_parent, dst_basename, pool);
142 /* Sanity checks: SRC and DST_PARENT are directories, and
143 DST_BASENAME doesn't already exist in DST_PARENT. */
144 SVN_ERR(svn_io_check_path(src, &kind, subpool));
145 if (kind != svn_node_dir)
146 return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL,
147 _("Source '%s' is not a directory"),
148 svn_dirent_local_style(src, pool));
150 SVN_ERR(svn_io_check_path(dst_parent, &kind, subpool));
151 if (kind != svn_node_dir)
152 return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL,
153 _("Destination '%s' is not a directory"),
154 svn_dirent_local_style(dst_parent, pool));
156 SVN_ERR(svn_io_check_path(dst_path, &kind, subpool));
158 /* Create the new directory. */
159 /* ### TODO: copy permissions (needs apr_file_attrs_get()) */
160 SVN_ERR(svn_io_make_dir_recursively(dst_path, pool));
162 /* Loop over the dirents in SRC. ('.' and '..' are auto-excluded) */
163 SVN_ERR(svn_io_dir_open(&this_dir, src, subpool));
165 for (status = apr_dir_read(&this_entry, flags, this_dir);
166 status == APR_SUCCESS;
167 status = apr_dir_read(&this_entry, flags, this_dir))
169 if ((this_entry.name[0] == '.')
170 && ((this_entry.name[1] == '\0')
171 || ((this_entry.name[1] == '.')
172 && (this_entry.name[2] == '\0'))))
178 const char *entryname_utf8;
181 SVN_ERR(cancel_func(cancel_baton));
183 SVN_ERR(entry_name_to_utf8(&entryname_utf8, this_entry.name,
185 if (this_entry.filetype == APR_REG) /* regular file */
187 SVN_ERR(hotcopy_io_dir_file_copy(skipped_p, src, dst_path,
188 entryname_utf8, subpool));
190 else if (this_entry.filetype == APR_LNK) /* symlink */
192 const char *src_target = svn_dirent_join(src, entryname_utf8,
194 const char *dst_target = svn_dirent_join(dst_path,
197 SVN_ERR(svn_io_copy_link(src_target, dst_target,
200 else if (this_entry.filetype == APR_DIR) /* recurse */
202 const char *src_target;
204 /* Prevent infinite recursion by filtering off our
205 newly created destination path. */
206 if (strcmp(src, dst_parent) == 0
207 && strcmp(entryname_utf8, dst_basename) == 0)
210 src_target = svn_dirent_join(src, entryname_utf8, subpool);
211 SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p,
220 /* ### support other APR node types someday?? */
225 if (! (APR_STATUS_IS_ENOENT(status)))
226 return svn_error_wrap_apr(status, _("Can't read directory '%s'"),
227 svn_dirent_local_style(src, pool));
229 status = apr_dir_close(this_dir);
231 return svn_error_wrap_apr(status, _("Error closing directory '%s'"),
232 svn_dirent_local_style(src, pool));
234 /* Free any memory used by recursion */
235 svn_pool_destroy(subpool);
240 /* Copy an un-packed revision or revprop file for revision REV from SRC_SUBDIR
241 * to DST_SUBDIR. Assume a sharding layout based on MAX_FILES_PER_DIR.
242 * Set *SKIPPED_P to FALSE only if the file was copied, do not change the
243 * value in *SKIPPED_P otherwise. SKIPPED_P may be NULL if not required.
244 * Use SCRATCH_POOL for temporary allocations. */
246 hotcopy_copy_shard_file(svn_boolean_t *skipped_p,
247 const char *src_subdir,
248 const char *dst_subdir,
250 int max_files_per_dir,
251 apr_pool_t *scratch_pool)
253 const char *src_subdir_shard = src_subdir,
254 *dst_subdir_shard = dst_subdir;
256 if (max_files_per_dir)
258 const char *shard = apr_psprintf(scratch_pool, "%ld",
259 rev / max_files_per_dir);
260 src_subdir_shard = svn_dirent_join(src_subdir, shard, scratch_pool);
261 dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool);
263 if (rev % max_files_per_dir == 0)
265 SVN_ERR(svn_io_make_dir_recursively(dst_subdir_shard, scratch_pool));
266 SVN_ERR(svn_io_copy_perms(dst_subdir, dst_subdir_shard,
271 SVN_ERR(hotcopy_io_dir_file_copy(skipped_p,
272 src_subdir_shard, dst_subdir_shard,
273 apr_psprintf(scratch_pool, "%ld", rev),
280 /* Copy a packed shard containing revision REV, and which contains
281 * MAX_FILES_PER_DIR revisions, from SRC_FS to DST_FS.
282 * Update *DST_MIN_UNPACKED_REV in case the shard is new in DST_FS.
283 * Do not re-copy data which already exists in DST_FS.
284 * Set *SKIPPED_P to FALSE only if at least one part of the shard
285 * was copied, do not change the value in *SKIPPED_P otherwise.
286 * SKIPPED_P may be NULL if not required.
287 * Use SCRATCH_POOL for temporary allocations. */
289 hotcopy_copy_packed_shard(svn_boolean_t *skipped_p,
290 svn_revnum_t *dst_min_unpacked_rev,
294 int max_files_per_dir,
295 apr_pool_t *scratch_pool)
297 const char *src_subdir;
298 const char *dst_subdir;
299 const char *packed_shard;
300 const char *src_subdir_packed_shard;
301 svn_revnum_t revprop_rev;
302 apr_pool_t *iterpool;
303 fs_fs_data_t *src_ffd = src_fs->fsap_data;
305 /* Copy the packed shard. */
306 src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, scratch_pool);
307 dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, scratch_pool);
308 packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD,
309 rev / max_files_per_dir);
310 src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard,
312 SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p, src_subdir_packed_shard,
313 dst_subdir, packed_shard,
314 TRUE /* copy_perms */,
315 NULL /* cancel_func */, NULL,
318 /* Copy revprops belonging to revisions in this pack. */
319 src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, scratch_pool);
320 dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, scratch_pool);
322 if ( src_ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT
323 || src_ffd->min_unpacked_rev < rev + max_files_per_dir)
325 /* copy unpacked revprops rev by rev */
326 iterpool = svn_pool_create(scratch_pool);
327 for (revprop_rev = rev;
328 revprop_rev < rev + max_files_per_dir;
331 svn_pool_clear(iterpool);
333 SVN_ERR(hotcopy_copy_shard_file(skipped_p, src_subdir, dst_subdir,
334 revprop_rev, max_files_per_dir,
337 svn_pool_destroy(iterpool);
341 /* revprop for revision 0 will never be packed */
343 SVN_ERR(hotcopy_copy_shard_file(skipped_p, src_subdir, dst_subdir,
344 0, max_files_per_dir,
347 /* packed revprops folder */
348 packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD,
349 rev / max_files_per_dir);
350 src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard,
352 SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p,
353 src_subdir_packed_shard,
354 dst_subdir, packed_shard,
355 TRUE /* copy_perms */,
356 NULL /* cancel_func */, NULL,
360 /* If necessary, update the min-unpacked rev file in the hotcopy. */
361 if (*dst_min_unpacked_rev < rev + max_files_per_dir)
363 *dst_min_unpacked_rev = rev + max_files_per_dir;
364 SVN_ERR(svn_fs_fs__write_min_unpacked_rev(dst_fs,
365 *dst_min_unpacked_rev,
372 /* Remove file PATH, if it exists - even if it is read-only.
373 * Use POOL for temporary allocations. */
375 hotcopy_remove_file(const char *path,
378 /* Make the rev file writable and remove it. */
379 SVN_ERR(svn_io_set_file_read_write(path, TRUE, pool));
380 SVN_ERR(svn_io_remove_file2(path, TRUE, pool));
386 /* Remove revision or revprop files between START_REV (inclusive) and
387 * END_REV (non-inclusive) from folder DST_SUBDIR in DST_FS. Assume
388 * sharding as per MAX_FILES_PER_DIR.
389 * Use SCRATCH_POOL for temporary allocations. */
391 hotcopy_remove_files(svn_fs_t *dst_fs,
392 const char *dst_subdir,
393 svn_revnum_t start_rev,
394 svn_revnum_t end_rev,
395 int max_files_per_dir,
396 apr_pool_t *scratch_pool)
399 const char *dst_subdir_shard;
401 apr_pool_t *iterpool;
403 /* Pre-compute paths for initial shard. */
404 shard = apr_psprintf(scratch_pool, "%ld", start_rev / max_files_per_dir);
405 dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool);
407 iterpool = svn_pool_create(scratch_pool);
408 for (rev = start_rev; rev < end_rev; rev++)
410 svn_pool_clear(iterpool);
412 /* If necessary, update paths for shard. */
413 if (rev != start_rev && rev % max_files_per_dir == 0)
415 shard = apr_psprintf(iterpool, "%ld", rev / max_files_per_dir);
416 dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool);
419 /* remove files for REV */
420 SVN_ERR(hotcopy_remove_file(svn_dirent_join(dst_subdir_shard,
421 apr_psprintf(iterpool,
427 svn_pool_destroy(iterpool);
432 /* Remove revisions between START_REV (inclusive) and END_REV (non-inclusive)
433 * from DST_FS. Assume sharding as per MAX_FILES_PER_DIR.
434 * Use SCRATCH_POOL for temporary allocations. */
436 hotcopy_remove_rev_files(svn_fs_t *dst_fs,
437 svn_revnum_t start_rev,
438 svn_revnum_t end_rev,
439 int max_files_per_dir,
440 apr_pool_t *scratch_pool)
442 SVN_ERR_ASSERT(start_rev <= end_rev);
443 SVN_ERR(hotcopy_remove_files(dst_fs,
444 svn_dirent_join(dst_fs->path,
448 max_files_per_dir, scratch_pool));
453 /* Remove revision properties between START_REV (inclusive) and END_REV
454 * (non-inclusive) from DST_FS. Assume sharding as per MAX_FILES_PER_DIR.
455 * Use SCRATCH_POOL for temporary allocations. Revision 0 revprops will
458 hotcopy_remove_revprop_files(svn_fs_t *dst_fs,
459 svn_revnum_t start_rev,
460 svn_revnum_t end_rev,
461 int max_files_per_dir,
462 apr_pool_t *scratch_pool)
464 SVN_ERR_ASSERT(start_rev <= end_rev);
466 /* don't delete rev 0 props */
467 SVN_ERR(hotcopy_remove_files(dst_fs,
468 svn_dirent_join(dst_fs->path,
471 start_rev ? start_rev : 1, end_rev,
472 max_files_per_dir, scratch_pool));
477 /* Verify that DST_FS is a suitable destination for an incremental
478 * hotcopy from SRC_FS. */
480 hotcopy_incremental_check_preconditions(svn_fs_t *src_fs,
484 fs_fs_data_t *src_ffd = src_fs->fsap_data;
485 fs_fs_data_t *dst_ffd = dst_fs->fsap_data;
487 /* We only support incremental hotcopy between the same format. */
488 if (src_ffd->format != dst_ffd->format)
489 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
490 _("The FSFS format (%d) of the hotcopy source does not match the "
491 "FSFS format (%d) of the hotcopy destination; please upgrade "
492 "both repositories to the same format"),
493 src_ffd->format, dst_ffd->format);
495 /* Make sure the UUID of source and destination match up.
496 * We don't want to copy over a different repository. */
497 if (strcmp(src_fs->uuid, dst_fs->uuid) != 0)
498 return svn_error_create(SVN_ERR_RA_UUID_MISMATCH, NULL,
499 _("The UUID of the hotcopy source does "
500 "not match the UUID of the hotcopy "
503 /* Also require same shard size. */
504 if (src_ffd->max_files_per_dir != dst_ffd->max_files_per_dir)
505 return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
506 _("The sharding layout configuration "
507 "of the hotcopy source does not match "
508 "the sharding layout configuration of "
509 "the hotcopy destination"));
513 /* Remove folder PATH. Ignore errors due to the sub-tree not being empty.
514 * CANCEL_FUNC and CANCEL_BATON do the usual thing.
515 * Use POOL for temporary allocations.
518 remove_folder(const char *path,
519 svn_cancel_func_t cancel_func,
523 svn_error_t *err = svn_io_remove_dir2(path, TRUE,
524 cancel_func, cancel_baton, pool);
526 if (err && APR_STATUS_IS_ENOTEMPTY(err->apr_err))
528 svn_error_clear(err);
532 return svn_error_trace(err);
535 /* Copy the revision and revprop files (possibly sharded / packed) from
536 * SRC_FS to DST_FS. Do not re-copy data which already exists in DST_FS.
537 * When copying packed or unpacked shards, checkpoint the result in DST_FS
538 * for every shard by updating the 'current' file if necessary. Assume
539 * the >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT filesystem format without
540 * global next-ID counters. Indicate progress via the optional NOTIFY_FUNC
541 * callback using NOTIFY_BATON. Use POOL for temporary allocations.
544 hotcopy_revisions(svn_fs_t *src_fs,
546 svn_revnum_t src_youngest,
547 svn_revnum_t dst_youngest,
548 svn_boolean_t incremental,
549 const char *src_revs_dir,
550 const char *dst_revs_dir,
551 const char *src_revprops_dir,
552 const char *dst_revprops_dir,
553 svn_fs_hotcopy_notify_t notify_func,
555 svn_cancel_func_t cancel_func,
559 fs_fs_data_t *src_ffd = src_fs->fsap_data;
560 fs_fs_data_t *dst_ffd = dst_fs->fsap_data;
561 int max_files_per_dir = src_ffd->max_files_per_dir;
562 svn_revnum_t src_min_unpacked_rev;
563 svn_revnum_t dst_min_unpacked_rev;
565 apr_pool_t *iterpool;
567 /* Copy the min unpacked rev, and read its value. */
568 if (src_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
570 SVN_ERR(svn_fs_fs__read_min_unpacked_rev(&src_min_unpacked_rev,
572 SVN_ERR(svn_fs_fs__read_min_unpacked_rev(&dst_min_unpacked_rev,
575 /* We only support packs coming from the hotcopy source.
576 * The destination should not be packed independently from
577 * the source. This also catches the case where users accidentally
578 * swap the source and destination arguments. */
579 if (src_min_unpacked_rev < dst_min_unpacked_rev)
580 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
581 _("The hotcopy destination already contains "
582 "more packed revisions (%lu) than the "
583 "hotcopy source contains (%lu)"),
584 dst_min_unpacked_rev - 1,
585 src_min_unpacked_rev - 1);
587 SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path,
588 PATH_MIN_UNPACKED_REV, pool));
592 src_min_unpacked_rev = 0;
593 dst_min_unpacked_rev = 0;
597 SVN_ERR(cancel_func(cancel_baton));
600 * Copy the necessary rev files.
603 iterpool = svn_pool_create(pool);
604 /* First, copy packed shards. */
605 for (rev = 0; rev < src_min_unpacked_rev; rev += max_files_per_dir)
607 svn_boolean_t skipped = TRUE;
608 svn_revnum_t pack_end_rev;
610 svn_pool_clear(iterpool);
613 SVN_ERR(cancel_func(cancel_baton));
615 /* Copy the packed shard. */
616 SVN_ERR(hotcopy_copy_packed_shard(&skipped, &dst_min_unpacked_rev,
618 rev, max_files_per_dir,
621 pack_end_rev = rev + max_files_per_dir - 1;
623 /* Whenever this pack did not previously exist in the destination,
624 * update 'current' to the most recent packed rev (so readers can see
625 * new revisions which arrived in this pack). */
626 if (pack_end_rev > dst_youngest)
628 SVN_ERR(svn_fs_fs__write_current(dst_fs, pack_end_rev, 0, 0,
632 /* When notifying about packed shards, make things simpler by either
633 * reporting a full revision range, i.e [pack start, pack end] or
634 * reporting nothing. There is one case when this approach might not
635 * be exact (incremental hotcopy with a pack replacing last unpacked
636 * revisions), but generally this is good enough. */
637 if (notify_func && !skipped)
638 notify_func(notify_baton, rev, pack_end_rev, iterpool);
640 /* Remove revision files which are now packed. */
643 SVN_ERR(hotcopy_remove_rev_files(dst_fs, rev,
644 rev + max_files_per_dir,
645 max_files_per_dir, iterpool));
646 if (dst_ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
647 SVN_ERR(hotcopy_remove_revprop_files(dst_fs, rev,
648 rev + max_files_per_dir,
653 /* Now that all revisions have moved into the pack, the original
654 * rev dir can be removed. */
655 SVN_ERR(remove_folder(svn_fs_fs__path_rev_shard(dst_fs, rev, iterpool),
656 cancel_func, cancel_baton, iterpool));
657 if (rev > 0 && dst_ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
658 SVN_ERR(remove_folder(svn_fs_fs__path_revprops_shard(dst_fs, rev,
660 cancel_func, cancel_baton, iterpool));
664 SVN_ERR(cancel_func(cancel_baton));
666 SVN_ERR_ASSERT(rev == src_min_unpacked_rev);
667 SVN_ERR_ASSERT(src_min_unpacked_rev == dst_min_unpacked_rev);
669 /* Now, copy pairs of non-packed revisions and revprop files.
670 * If necessary, update 'current' after copying all files from a shard. */
671 for (; rev <= src_youngest; rev++)
673 svn_boolean_t skipped = TRUE;
675 svn_pool_clear(iterpool);
678 SVN_ERR(cancel_func(cancel_baton));
680 /* Copying non-packed revisions is racy in case the source repository is
681 * being packed concurrently with this hotcopy operation. The race can
682 * happen with FS formats prior to SVN_FS_FS__MIN_PACK_LOCK_FORMAT that
683 * support packed revisions. With the pack lock, however, the race is
684 * impossible, because hotcopy and pack operations block each other.
686 * We assume that all revisions coming after 'min-unpacked-rev' really
687 * are unpacked and that's not necessarily true with concurrent packing.
688 * Don't try to be smart in this edge case, because handling it properly
689 * might require copying *everything* from the start. Just abort the
690 * hotcopy with an ENOENT (revision file moved to a pack, so it is no
691 * longer where we expect it to be). */
693 /* Copy the rev file. */
694 SVN_ERR(hotcopy_copy_shard_file(&skipped,
695 src_revs_dir, dst_revs_dir, rev,
698 /* Copy the revprop file. */
699 SVN_ERR(hotcopy_copy_shard_file(&skipped,
700 src_revprops_dir, dst_revprops_dir,
701 rev, max_files_per_dir,
704 /* Whenever this revision did not previously exist in the destination,
705 * checkpoint the progress via 'current' (do that once per full shard
706 * in order not to slow things down). */
707 if (rev > dst_youngest)
709 if (max_files_per_dir && (rev % max_files_per_dir == 0))
711 SVN_ERR(svn_fs_fs__write_current(dst_fs, rev, 0, 0,
716 if (notify_func && !skipped)
717 notify_func(notify_baton, rev, rev, iterpool);
719 svn_pool_destroy(iterpool);
721 /* We assume that all revisions were copied now, i.e. we didn't exit the
722 * above loop early. 'rev' was last incremented during exit of the loop. */
723 SVN_ERR_ASSERT(rev == src_youngest + 1);
728 /* Shortcut for the revision and revprop copying for old (1 or 2) format
729 * filesystems without sharding and packing. Copy the non-sharded revision
730 * and revprop files from SRC_FS to DST_FS. Do not re-copy data which
731 * already exists in DST_FS. Do not somehow checkpoint the results in
732 * the 'current' file in DST_FS. Indicate progress via the optional
733 * NOTIFY_FUNC callback using NOTIFY_BATON. Use POOL for temporary
734 * allocations. Also see hotcopy_revisions().
737 hotcopy_revisions_old(svn_fs_t *src_fs,
739 svn_revnum_t src_youngest,
740 const char *src_revs_dir,
741 const char *dst_revs_dir,
742 const char *src_revprops_dir,
743 const char *dst_revprops_dir,
744 svn_fs_hotcopy_notify_t notify_func,
746 svn_cancel_func_t cancel_func,
750 apr_pool_t *iterpool = svn_pool_create(pool);
753 for (rev = 0; rev <= src_youngest; rev++)
755 svn_boolean_t skipped = TRUE;
757 svn_pool_clear(iterpool);
760 SVN_ERR(cancel_func(cancel_baton));
762 SVN_ERR(hotcopy_io_dir_file_copy(&skipped, src_revs_dir, dst_revs_dir,
763 apr_psprintf(iterpool, "%ld", rev),
765 SVN_ERR(hotcopy_io_dir_file_copy(&skipped, src_revprops_dir,
767 apr_psprintf(iterpool, "%ld", rev),
770 if (notify_func && !skipped)
771 notify_func(notify_baton, rev, rev, iterpool);
773 svn_pool_destroy(iterpool);
778 /* Baton for hotcopy_body(). */
779 struct hotcopy_body_baton {
782 svn_boolean_t incremental;
783 svn_fs_hotcopy_notify_t notify_func;
785 svn_cancel_func_t cancel_func;
789 /* Perform a hotcopy, either normal or incremental.
791 * Normal hotcopy assumes that the destination exists as an empty
792 * directory. It behaves like an incremental hotcopy except that
793 * none of the copied files already exist in the destination.
795 * An incremental hotcopy copies only changed or new files to the destination,
796 * and removes files from the destination no longer present in the source.
797 * While the incremental hotcopy is running, readers should still be able
798 * to access the destination repository without error and should not see
799 * revisions currently in progress of being copied. Readers are able to see
800 * new fully copied revisions even if the entire incremental hotcopy procedure
801 * has not yet completed.
803 * Writers are blocked out completely during the entire incremental hotcopy
804 * process to ensure consistency. This function assumes that the repository
805 * write-lock is held.
808 hotcopy_body(void *baton, apr_pool_t *pool)
810 struct hotcopy_body_baton *hbb = baton;
811 svn_fs_t *src_fs = hbb->src_fs;
812 fs_fs_data_t *src_ffd = src_fs->fsap_data;
813 svn_fs_t *dst_fs = hbb->dst_fs;
814 fs_fs_data_t *dst_ffd = dst_fs->fsap_data;
815 svn_boolean_t incremental = hbb->incremental;
816 svn_fs_hotcopy_notify_t notify_func = hbb->notify_func;
817 void* notify_baton = hbb->notify_baton;
818 svn_cancel_func_t cancel_func = hbb->cancel_func;
819 void* cancel_baton = hbb->cancel_baton;
820 svn_revnum_t src_youngest;
821 apr_uint64_t src_next_node_id;
822 apr_uint64_t src_next_copy_id;
823 svn_revnum_t dst_youngest;
824 const char *src_revprops_dir;
825 const char *dst_revprops_dir;
826 const char *src_revs_dir;
827 const char *dst_revs_dir;
828 const char *src_subdir;
829 const char *dst_subdir;
830 svn_node_kind_t kind;
832 /* Try to copy the config.
834 * ### We try copying the config file before doing anything else,
835 * ### because higher layers will abort the hotcopy if we throw
836 * ### an error from this function, and that renders the hotcopy
837 * ### unusable anyway. */
838 if (src_ffd->format >= SVN_FS_FS__MIN_CONFIG_FILE)
842 err = svn_io_dir_file_copy(src_fs->path, dst_fs->path, PATH_CONFIG,
846 if (APR_STATUS_IS_ENOENT(err->apr_err))
848 /* 1.6.0 to 1.6.11 did not copy the configuration file during
849 * hotcopy. So if we're hotcopying a repository which has been
850 * created as a hotcopy itself, it's possible that fsfs.conf
851 * does not exist. Ask the user to re-create it.
853 * ### It would be nice to make this a non-fatal error,
854 * ### but this function does not get an svn_fs_t object
855 * ### so we have no way of just printing a warning via
856 * ### the fs->warning() callback. */
858 const char *src_abspath;
859 const char *dst_abspath;
860 const char *config_relpath;
863 config_relpath = svn_dirent_join(src_fs->path, PATH_CONFIG, pool);
864 err2 = svn_dirent_get_absolute(&src_abspath, src_fs->path, pool);
866 return svn_error_trace(svn_error_compose_create(err, err2));
867 err2 = svn_dirent_get_absolute(&dst_abspath, dst_fs->path, pool);
869 return svn_error_trace(svn_error_compose_create(err, err2));
871 /* ### hack: strip off the 'db/' directory from paths so
872 * ### they make sense to the user */
873 src_abspath = svn_dirent_dirname(src_abspath, pool);
874 dst_abspath = svn_dirent_dirname(dst_abspath, pool);
876 return svn_error_quick_wrapf(err,
877 _("Failed to create hotcopy at '%s'. "
878 "The file '%s' is missing from the source "
879 "repository. Please create this file, for "
880 "instance by running 'svnadmin upgrade %s'"),
881 dst_abspath, config_relpath, src_abspath);
884 return svn_error_trace(err);
889 SVN_ERR(cancel_func(cancel_baton));
891 /* Find the youngest revision in the source and destination.
892 * We only support hotcopies from sources with an equal or greater amount
893 * of revisions than the destination.
894 * This also catches the case where users accidentally swap the
895 * source and destination arguments. */
896 SVN_ERR(svn_fs_fs__read_current(&src_youngest, &src_next_node_id,
897 &src_next_copy_id, src_fs, pool));
900 SVN_ERR(svn_fs_fs__youngest_rev(&dst_youngest, dst_fs, pool));
901 if (src_youngest < dst_youngest)
902 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
903 _("The hotcopy destination already contains more revisions "
904 "(%lu) than the hotcopy source contains (%lu); are source "
905 "and destination swapped?"),
906 dst_youngest, src_youngest);
911 src_revs_dir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, pool);
912 dst_revs_dir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, pool);
913 src_revprops_dir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, pool);
914 dst_revprops_dir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, pool);
916 /* Ensure that the required folders exist in the destination
917 * before actually copying the revisions and revprops. */
918 SVN_ERR(svn_io_make_dir_recursively(dst_revs_dir, pool));
919 SVN_ERR(svn_io_make_dir_recursively(dst_revprops_dir, pool));
922 SVN_ERR(cancel_func(cancel_baton));
924 /* Split the logic for new and old FS formats. The latter is much simpler
925 * due to the absense of sharding and packing. However, it requires special
926 * care when updating the 'current' file (which contains not just the
927 * revision number, but also the next-ID counters). */
928 if (src_ffd->format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT)
930 SVN_ERR(hotcopy_revisions(src_fs, dst_fs, src_youngest, dst_youngest,
931 incremental, src_revs_dir, dst_revs_dir,
932 src_revprops_dir, dst_revprops_dir,
933 notify_func, notify_baton,
934 cancel_func, cancel_baton, pool));
935 SVN_ERR(svn_fs_fs__write_current(dst_fs, src_youngest, 0, 0, pool));
939 SVN_ERR(hotcopy_revisions_old(src_fs, dst_fs, src_youngest,
940 src_revs_dir, dst_revs_dir,
941 src_revprops_dir, dst_revprops_dir,
942 notify_func, notify_baton,
943 cancel_func, cancel_baton, pool));
944 SVN_ERR(svn_fs_fs__write_current(dst_fs, src_youngest, src_next_node_id,
945 src_next_copy_id, pool));
948 /* Replace the locks tree.
949 * This is racy in case readers are currently trying to list locks in
950 * the destination. However, we need to get rid of stale locks.
951 * This is the simplest way of doing this, so we accept this small race. */
952 dst_subdir = svn_dirent_join(dst_fs->path, PATH_LOCKS_DIR, pool);
953 SVN_ERR(svn_io_remove_dir2(dst_subdir, TRUE, cancel_func, cancel_baton,
955 src_subdir = svn_dirent_join(src_fs->path, PATH_LOCKS_DIR, pool);
956 SVN_ERR(svn_io_check_path(src_subdir, &kind, pool));
957 if (kind == svn_node_dir)
958 SVN_ERR(svn_io_copy_dir_recursively(src_subdir, dst_fs->path,
959 PATH_LOCKS_DIR, TRUE,
960 cancel_func, cancel_baton, pool));
962 /* Now copy the node-origins cache tree. */
963 src_subdir = svn_dirent_join(src_fs->path, PATH_NODE_ORIGINS_DIR, pool);
964 SVN_ERR(svn_io_check_path(src_subdir, &kind, pool));
965 if (kind == svn_node_dir)
966 SVN_ERR(hotcopy_io_copy_dir_recursively(NULL, src_subdir, dst_fs->path,
967 PATH_NODE_ORIGINS_DIR, TRUE,
968 cancel_func, cancel_baton, pool));
971 * NB: Data copied below is only read by writers, not readers.
972 * Writers are still locked out at this point.
975 if (dst_ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT)
977 /* Copy the rep cache and then remove entries for revisions
978 * that did not make it into the destination. */
979 src_subdir = svn_dirent_join(src_fs->path, REP_CACHE_DB_NAME, pool);
980 dst_subdir = svn_dirent_join(dst_fs->path, REP_CACHE_DB_NAME, pool);
981 SVN_ERR(svn_io_check_path(src_subdir, &kind, pool));
982 if (kind == svn_node_file)
984 SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, pool));
986 /* The source might have r/o flags set on it - which would be
987 carried over to the copy. */
988 SVN_ERR(svn_io_set_file_read_write(dst_subdir, FALSE, pool));
989 SVN_ERR(svn_fs_fs__del_rep_reference(dst_fs, src_youngest, pool));
993 /* Copy the txn-current file. */
994 if (dst_ffd->format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT)
995 SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path,
996 PATH_TXN_CURRENT, pool));
998 /* Hotcopied FS is complete. Stamp it with a format file. */
999 SVN_ERR(svn_fs_fs__write_format(dst_fs, TRUE, pool));
1001 return SVN_NO_ERROR;
1005 svn_fs_fs__hotcopy(svn_fs_t *src_fs,
1007 const char *src_path,
1008 const char *dst_path,
1009 svn_boolean_t incremental,
1010 svn_fs_hotcopy_notify_t notify_func,
1012 svn_cancel_func_t cancel_func,
1014 svn_mutex__t *common_pool_lock,
1016 apr_pool_t *common_pool)
1018 struct hotcopy_body_baton hbb;
1021 SVN_ERR(cancel_func(cancel_baton));
1023 SVN_ERR(svn_fs_fs__open(src_fs, src_path, pool));
1027 const char *dst_format_abspath;
1028 svn_node_kind_t dst_format_kind;
1030 /* Check destination format to be sure we know how to incrementally
1031 * hotcopy to the destination FS. */
1032 dst_format_abspath = svn_dirent_join(dst_path, PATH_FORMAT, pool);
1033 SVN_ERR(svn_io_check_path(dst_format_abspath, &dst_format_kind, pool));
1034 if (dst_format_kind == svn_node_none)
1036 /* No destination? Fallback to a non-incremental hotcopy. */
1037 incremental = FALSE;
1043 /* Check the existing repository. */
1044 SVN_ERR(svn_fs_fs__open(dst_fs, dst_path, pool));
1045 SVN_ERR(hotcopy_incremental_check_preconditions(src_fs, dst_fs, pool));
1047 SVN_ERR(svn_fs_fs__initialize_shared_data(dst_fs, common_pool_lock,
1048 pool, common_pool));
1049 SVN_ERR(svn_fs_fs__initialize_caches(dst_fs, pool));
1053 /* Start out with an empty destination using the same configuration
1055 fs_fs_data_t *src_ffd = src_fs->fsap_data;
1057 /* Create the DST_FS repository with the same layout as SRC_FS. */
1058 SVN_ERR(svn_fs_fs__create_file_tree(dst_fs, dst_path, src_ffd->format,
1059 src_ffd->max_files_per_dir,
1060 src_ffd->use_log_addressing,
1063 /* Copy the UUID. Hotcopy destination receives a new instance ID, but
1064 * has the same filesystem UUID as the source. */
1065 SVN_ERR(svn_fs_fs__set_uuid(dst_fs, src_fs->uuid, NULL, pool));
1067 /* Remove revision 0 contents. Otherwise, it may not get overwritten
1068 * due to having a newer timestamp. */
1069 SVN_ERR(hotcopy_remove_file(svn_fs_fs__path_rev(dst_fs, 0, pool),
1071 SVN_ERR(hotcopy_remove_file(svn_fs_fs__path_revprops(dst_fs, 0, pool),
1074 SVN_ERR(svn_fs_fs__initialize_shared_data(dst_fs, common_pool_lock,
1075 pool, common_pool));
1076 SVN_ERR(svn_fs_fs__initialize_caches(dst_fs, pool));
1080 SVN_ERR(cancel_func(cancel_baton));
1082 hbb.src_fs = src_fs;
1083 hbb.dst_fs = dst_fs;
1084 hbb.incremental = incremental;
1085 hbb.notify_func = notify_func;
1086 hbb.notify_baton = notify_baton;
1087 hbb.cancel_func = cancel_func;
1088 hbb.cancel_baton = cancel_baton;
1090 /* Lock the destination in the incremental mode. For a non-incremental
1091 * hotcopy, don't take any locks. In that case the destination cannot be
1092 * opened until the hotcopy finishes, and we don't have to worry about
1095 SVN_ERR(svn_fs_fs__with_all_locks(dst_fs, hotcopy_body, &hbb, pool));
1097 SVN_ERR(hotcopy_body(&hbb, pool));
1099 return SVN_NO_ERROR;