1 /* fs_fs.c --- filesystem operations specific to fs_fs
3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
20 * ====================================================================
30 #include <apr_general.h>
31 #include <apr_pools.h>
32 #include <apr_file_io.h>
37 #include <apr_strings.h>
38 #include <apr_thread_mutex.h>
40 #include "svn_pools.h"
42 #include "svn_dirent_uri.h"
45 #include "svn_props.h"
46 #include "svn_sorts.h"
47 #include "svn_string.h"
49 #include "svn_mergeinfo.h"
50 #include "svn_config.h"
51 #include "svn_ctype.h"
52 #include "svn_version.h"
60 #include "rep-cache.h"
61 #include "temp_serializer.h"
63 #include "private/svn_string_private.h"
64 #include "private/svn_fs_util.h"
65 #include "private/svn_subr_private.h"
66 #include "private/svn_delta_private.h"
67 #include "../libsvn_fs/fs-loader.h"
69 #include "svn_private_config.h"
70 #include "temp_serializer.h"
72 /* An arbitrary maximum path length, so clients can't run us out of memory
73 * by giving us arbitrarily large paths. */
74 #define FSFS_MAX_PATH_LEN 4096
76 /* The default maximum number of files per directory to store in the
77 rev and revprops directory. The number below is somewhat arbitrary,
78 and can be overridden by defining the macro while compiling; the
79 figure of 1000 is reasonable for VFAT filesystems, which are by far
80 the worst performers in this area. */
81 #ifndef SVN_FS_FS_DEFAULT_MAX_FILES_PER_DIR
82 #define SVN_FS_FS_DEFAULT_MAX_FILES_PER_DIR 1000
85 /* Begin deltification after a node history exceeded this this limit.
86 Useful values are 4 to 64 with 16 being a good compromise between
87 computational overhead and repository size savings.
88 Should be a power of 2.
89 Values < 2 will result in standard skip-delta behavior. */
90 #define SVN_FS_FS_MAX_LINEAR_DELTIFICATION 16
92 /* Finding a deltification base takes operations proportional to the
93 number of changes being skipped. To prevent exploding runtime
94 during commits, limit the deltification range to this value.
95 Should be a power of 2 minus one.
96 Values < 1 disable deltification. */
97 #define SVN_FS_FS_MAX_DELTIFICATION_WALK 1023
99 /* Give writing processes 10 seconds to replace an existing revprop
100 file with a new one. After that time, we assume that the writing
101 process got aborted and that we have re-read revprops. */
102 #define REVPROP_CHANGE_TIMEOUT (10 * 1000000)
104 /* The following are names of atomics that will be used to communicate
105 * revprop updates across all processes on this machine. */
106 #define ATOMIC_REVPROP_GENERATION "rev-prop-generation"
107 #define ATOMIC_REVPROP_TIMEOUT "rev-prop-timeout"
108 #define ATOMIC_REVPROP_NAMESPACE "rev-prop-atomics"
110 /* Following are defines that specify the textual elements of the
111 native filesystem directories and revision files. */
113 /* Headers used to describe node-revision in the revision file. */
114 #define HEADER_ID "id"
115 #define HEADER_TYPE "type"
116 #define HEADER_COUNT "count"
117 #define HEADER_PROPS "props"
118 #define HEADER_TEXT "text"
119 #define HEADER_CPATH "cpath"
120 #define HEADER_PRED "pred"
121 #define HEADER_COPYFROM "copyfrom"
122 #define HEADER_COPYROOT "copyroot"
123 #define HEADER_FRESHTXNRT "is-fresh-txn-root"
124 #define HEADER_MINFO_HERE "minfo-here"
125 #define HEADER_MINFO_CNT "minfo-cnt"
127 /* Kinds that a change can be. */
128 #define ACTION_MODIFY "modify"
129 #define ACTION_ADD "add"
130 #define ACTION_DELETE "delete"
131 #define ACTION_REPLACE "replace"
132 #define ACTION_RESET "reset"
134 /* True and False flags. */
135 #define FLAG_TRUE "true"
136 #define FLAG_FALSE "false"
138 /* Kinds that a node-rev can be. */
139 #define KIND_FILE "file"
140 #define KIND_DIR "dir"
142 /* Kinds of representation. */
143 #define REP_PLAIN "PLAIN"
144 #define REP_DELTA "DELTA"
148 To avoid opening and closing the rev-files all the time, it would
149 probably be advantageous to keep each rev-file open for the
150 lifetime of the transaction object. I'll leave that as a later
151 optimization for now.
153 I didn't keep track of pool lifetimes at all in this code. There
154 are likely some errors because of that.
158 /* The vtable associated with an open transaction object. */
159 static txn_vtable_t txn_vtable = {
160 svn_fs_fs__commit_txn,
161 svn_fs_fs__abort_txn,
163 svn_fs_fs__txn_proplist,
164 svn_fs_fs__change_txn_prop,
166 svn_fs_fs__change_txn_props
172 read_min_unpacked_rev(svn_revnum_t *min_unpacked_rev,
177 update_min_unpacked_rev(svn_fs_t *fs, apr_pool_t *pool);
180 get_youngest(svn_revnum_t *youngest_p, const char *fs_path, apr_pool_t *pool);
183 verify_walker(representation_t *rep,
186 apr_pool_t *scratch_pool);
188 /* Pathname helper functions */
190 /* Return TRUE is REV is packed in FS, FALSE otherwise. */
192 is_packed_rev(svn_fs_t *fs, svn_revnum_t rev)
194 fs_fs_data_t *ffd = fs->fsap_data;
196 return (rev < ffd->min_unpacked_rev);
199 /* Return TRUE is REV is packed in FS, FALSE otherwise. */
201 is_packed_revprop(svn_fs_t *fs, svn_revnum_t rev)
203 fs_fs_data_t *ffd = fs->fsap_data;
205 /* rev 0 will not be packed */
206 return (rev < ffd->min_unpacked_rev)
208 && (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT);
212 path_format(svn_fs_t *fs, apr_pool_t *pool)
214 return svn_dirent_join(fs->path, PATH_FORMAT, pool);
217 static APR_INLINE const char *
218 path_uuid(svn_fs_t *fs, apr_pool_t *pool)
220 return svn_dirent_join(fs->path, PATH_UUID, pool);
224 svn_fs_fs__path_current(svn_fs_t *fs, apr_pool_t *pool)
226 return svn_dirent_join(fs->path, PATH_CURRENT, pool);
229 static APR_INLINE const char *
230 path_txn_current(svn_fs_t *fs, apr_pool_t *pool)
232 return svn_dirent_join(fs->path, PATH_TXN_CURRENT, pool);
235 static APR_INLINE const char *
236 path_txn_current_lock(svn_fs_t *fs, apr_pool_t *pool)
238 return svn_dirent_join(fs->path, PATH_TXN_CURRENT_LOCK, pool);
241 static APR_INLINE const char *
242 path_lock(svn_fs_t *fs, apr_pool_t *pool)
244 return svn_dirent_join(fs->path, PATH_LOCK_FILE, pool);
248 path_revprop_generation(svn_fs_t *fs, apr_pool_t *pool)
250 return svn_dirent_join(fs->path, PATH_REVPROP_GENERATION, pool);
254 path_rev_packed(svn_fs_t *fs, svn_revnum_t rev, const char *kind,
257 fs_fs_data_t *ffd = fs->fsap_data;
259 assert(ffd->max_files_per_dir);
260 assert(is_packed_rev(fs, rev));
262 return svn_dirent_join_many(pool, fs->path, PATH_REVS_DIR,
264 "%ld" PATH_EXT_PACKED_SHARD,
265 rev / ffd->max_files_per_dir),
270 path_rev_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool)
272 fs_fs_data_t *ffd = fs->fsap_data;
274 assert(ffd->max_files_per_dir);
275 return svn_dirent_join_many(pool, fs->path, PATH_REVS_DIR,
276 apr_psprintf(pool, "%ld",
277 rev / ffd->max_files_per_dir),
282 path_rev(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool)
284 fs_fs_data_t *ffd = fs->fsap_data;
286 assert(! is_packed_rev(fs, rev));
288 if (ffd->max_files_per_dir)
290 return svn_dirent_join(path_rev_shard(fs, rev, pool),
291 apr_psprintf(pool, "%ld", rev),
295 return svn_dirent_join_many(pool, fs->path, PATH_REVS_DIR,
296 apr_psprintf(pool, "%ld", rev), NULL);
300 svn_fs_fs__path_rev_absolute(const char **path,
305 fs_fs_data_t *ffd = fs->fsap_data;
307 if (ffd->format < SVN_FS_FS__MIN_PACKED_FORMAT
308 || ! is_packed_rev(fs, rev))
310 *path = path_rev(fs, rev, pool);
314 *path = path_rev_packed(fs, rev, PATH_PACKED, pool);
321 path_revprops_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool)
323 fs_fs_data_t *ffd = fs->fsap_data;
325 assert(ffd->max_files_per_dir);
326 return svn_dirent_join_many(pool, fs->path, PATH_REVPROPS_DIR,
327 apr_psprintf(pool, "%ld",
328 rev / ffd->max_files_per_dir),
333 path_revprops_pack_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool)
335 fs_fs_data_t *ffd = fs->fsap_data;
337 assert(ffd->max_files_per_dir);
338 return svn_dirent_join_many(pool, fs->path, PATH_REVPROPS_DIR,
339 apr_psprintf(pool, "%ld" PATH_EXT_PACKED_SHARD,
340 rev / ffd->max_files_per_dir),
345 path_revprops(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool)
347 fs_fs_data_t *ffd = fs->fsap_data;
349 if (ffd->max_files_per_dir)
351 return svn_dirent_join(path_revprops_shard(fs, rev, pool),
352 apr_psprintf(pool, "%ld", rev),
356 return svn_dirent_join_many(pool, fs->path, PATH_REVPROPS_DIR,
357 apr_psprintf(pool, "%ld", rev), NULL);
360 static APR_INLINE const char *
361 path_txn_dir(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool)
363 SVN_ERR_ASSERT_NO_RETURN(txn_id != NULL);
364 return svn_dirent_join_many(pool, fs->path, PATH_TXNS_DIR,
365 apr_pstrcat(pool, txn_id, PATH_EXT_TXN,
370 /* Return the name of the sha1->rep mapping file in transaction TXN_ID
371 * within FS for the given SHA1 checksum. Use POOL for allocations.
373 static APR_INLINE const char *
374 path_txn_sha1(svn_fs_t *fs, const char *txn_id, svn_checksum_t *sha1,
377 return svn_dirent_join(path_txn_dir(fs, txn_id, pool),
378 svn_checksum_to_cstring(sha1, pool),
382 static APR_INLINE const char *
383 path_txn_changes(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool)
385 return svn_dirent_join(path_txn_dir(fs, txn_id, pool), PATH_CHANGES, pool);
388 static APR_INLINE const char *
389 path_txn_props(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool)
391 return svn_dirent_join(path_txn_dir(fs, txn_id, pool), PATH_TXN_PROPS, pool);
394 static APR_INLINE const char *
395 path_txn_next_ids(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool)
397 return svn_dirent_join(path_txn_dir(fs, txn_id, pool), PATH_NEXT_IDS, pool);
400 static APR_INLINE const char *
401 path_min_unpacked_rev(svn_fs_t *fs, apr_pool_t *pool)
403 return svn_dirent_join(fs->path, PATH_MIN_UNPACKED_REV, pool);
407 static APR_INLINE const char *
408 path_txn_proto_rev(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool)
410 fs_fs_data_t *ffd = fs->fsap_data;
411 if (ffd->format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT)
412 return svn_dirent_join_many(pool, fs->path, PATH_TXN_PROTOS_DIR,
413 apr_pstrcat(pool, txn_id, PATH_EXT_REV,
417 return svn_dirent_join(path_txn_dir(fs, txn_id, pool), PATH_REV, pool);
420 static APR_INLINE const char *
421 path_txn_proto_rev_lock(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool)
423 fs_fs_data_t *ffd = fs->fsap_data;
424 if (ffd->format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT)
425 return svn_dirent_join_many(pool, fs->path, PATH_TXN_PROTOS_DIR,
426 apr_pstrcat(pool, txn_id, PATH_EXT_REV_LOCK,
430 return svn_dirent_join(path_txn_dir(fs, txn_id, pool), PATH_REV_LOCK,
435 path_txn_node_rev(svn_fs_t *fs, const svn_fs_id_t *id, apr_pool_t *pool)
437 const char *txn_id = svn_fs_fs__id_txn_id(id);
438 const char *node_id = svn_fs_fs__id_node_id(id);
439 const char *copy_id = svn_fs_fs__id_copy_id(id);
440 const char *name = apr_psprintf(pool, PATH_PREFIX_NODE "%s.%s",
443 return svn_dirent_join(path_txn_dir(fs, txn_id, pool), name, pool);
446 static APR_INLINE const char *
447 path_txn_node_props(svn_fs_t *fs, const svn_fs_id_t *id, apr_pool_t *pool)
449 return apr_pstrcat(pool, path_txn_node_rev(fs, id, pool), PATH_EXT_PROPS,
453 static APR_INLINE const char *
454 path_txn_node_children(svn_fs_t *fs, const svn_fs_id_t *id, apr_pool_t *pool)
456 return apr_pstrcat(pool, path_txn_node_rev(fs, id, pool),
457 PATH_EXT_CHILDREN, (char *)NULL);
460 static APR_INLINE const char *
461 path_node_origin(svn_fs_t *fs, const char *node_id, apr_pool_t *pool)
463 size_t len = strlen(node_id);
464 const char *node_id_minus_last_char =
465 (len == 1) ? "0" : apr_pstrmemdup(pool, node_id, len - 1);
466 return svn_dirent_join_many(pool, fs->path, PATH_NODE_ORIGINS_DIR,
467 node_id_minus_last_char, NULL);
470 static APR_INLINE const char *
471 path_and_offset_of(apr_file_t *file, apr_pool_t *pool)
474 apr_off_t offset = 0;
476 if (apr_file_name_get(&path, file) != APR_SUCCESS)
479 if (apr_file_seek(file, APR_CUR, &offset) != APR_SUCCESS)
482 return apr_psprintf(pool, "%s:%" APR_OFF_T_FMT, path, offset);
487 /* Functions for working with shared transaction data. */
489 /* Return the transaction object for transaction TXN_ID from the
490 transaction list of filesystem FS (which must already be locked via the
491 txn_list_lock mutex). If the transaction does not exist in the list,
492 then create a new transaction object and return it (if CREATE_NEW is
493 true) or return NULL (otherwise). */
494 static fs_fs_shared_txn_data_t *
495 get_shared_txn(svn_fs_t *fs, const char *txn_id, svn_boolean_t create_new)
497 fs_fs_data_t *ffd = fs->fsap_data;
498 fs_fs_shared_data_t *ffsd = ffd->shared;
499 fs_fs_shared_txn_data_t *txn;
501 for (txn = ffsd->txns; txn; txn = txn->next)
502 if (strcmp(txn->txn_id, txn_id) == 0)
505 if (txn || !create_new)
508 /* Use the transaction object from the (single-object) freelist,
509 if one is available, or otherwise create a new object. */
512 txn = ffsd->free_txn;
513 ffsd->free_txn = NULL;
517 apr_pool_t *subpool = svn_pool_create(ffsd->common_pool);
518 txn = apr_palloc(subpool, sizeof(*txn));
522 assert(strlen(txn_id) < sizeof(txn->txn_id));
523 apr_cpystrn(txn->txn_id, txn_id, sizeof(txn->txn_id));
524 txn->being_written = FALSE;
526 /* Link this transaction into the head of the list. We will typically
527 be dealing with only one active transaction at a time, so it makes
528 sense for searches through the transaction list to look at the
529 newest transactions first. */
530 txn->next = ffsd->txns;
536 /* Free the transaction object for transaction TXN_ID, and remove it
537 from the transaction list of filesystem FS (which must already be
538 locked via the txn_list_lock mutex). Do nothing if the transaction
541 free_shared_txn(svn_fs_t *fs, const char *txn_id)
543 fs_fs_data_t *ffd = fs->fsap_data;
544 fs_fs_shared_data_t *ffsd = ffd->shared;
545 fs_fs_shared_txn_data_t *txn, *prev = NULL;
547 for (txn = ffsd->txns; txn; prev = txn, txn = txn->next)
548 if (strcmp(txn->txn_id, txn_id) == 0)
555 prev->next = txn->next;
557 ffsd->txns = txn->next;
559 /* As we typically will be dealing with one transaction after another,
560 we will maintain a single-object free list so that we can hopefully
561 keep reusing the same transaction object. */
563 ffsd->free_txn = txn;
565 svn_pool_destroy(txn->pool);
569 /* Obtain a lock on the transaction list of filesystem FS, call BODY
570 with FS, BATON, and POOL, and then unlock the transaction list.
571 Return what BODY returned. */
573 with_txnlist_lock(svn_fs_t *fs,
574 svn_error_t *(*body)(svn_fs_t *fs,
580 fs_fs_data_t *ffd = fs->fsap_data;
581 fs_fs_shared_data_t *ffsd = ffd->shared;
583 SVN_MUTEX__WITH_LOCK(ffsd->txn_list_lock,
584 body(fs, baton, pool));
590 /* Get a lock on empty file LOCK_FILENAME, creating it in POOL. */
592 get_lock_on_filesystem(const char *lock_filename,
595 svn_error_t *err = svn_io_file_lock2(lock_filename, TRUE, FALSE, pool);
597 if (err && APR_STATUS_IS_ENOENT(err->apr_err))
599 /* No lock file? No big deal; these are just empty files
600 anyway. Create it and try again. */
601 svn_error_clear(err);
604 SVN_ERR(svn_io_file_create(lock_filename, "", pool));
605 SVN_ERR(svn_io_file_lock2(lock_filename, TRUE, FALSE, pool));
608 return svn_error_trace(err);
611 /* Reset the HAS_WRITE_LOCK member in the FFD given as BATON_VOID.
612 When registered with the pool holding the lock on the lock file,
613 this makes sure the flag gets reset just before we release the lock. */
615 reset_lock_flag(void *baton_void)
617 fs_fs_data_t *ffd = baton_void;
618 ffd->has_write_lock = FALSE;
622 /* Obtain a write lock on the file LOCK_FILENAME (protecting with
623 LOCK_MUTEX if APR is threaded) in a subpool of POOL, call BODY with
624 BATON and that subpool, destroy the subpool (releasing the write
625 lock) and return what BODY returned. If IS_GLOBAL_LOCK is set,
626 set the HAS_WRITE_LOCK flag while we keep the write lock. */
628 with_some_lock_file(svn_fs_t *fs,
629 svn_error_t *(*body)(void *baton,
632 const char *lock_filename,
633 svn_boolean_t is_global_lock,
636 apr_pool_t *subpool = svn_pool_create(pool);
637 svn_error_t *err = get_lock_on_filesystem(lock_filename, subpool);
641 fs_fs_data_t *ffd = fs->fsap_data;
645 /* set the "got the lock" flag and register reset function */
646 apr_pool_cleanup_register(subpool,
649 apr_pool_cleanup_null);
650 ffd->has_write_lock = TRUE;
653 /* nobody else will modify the repo state
654 => read HEAD & pack info once */
655 if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
656 SVN_ERR(update_min_unpacked_rev(fs, pool));
657 SVN_ERR(get_youngest(&ffd->youngest_rev_cache, fs->path,
659 err = body(baton, subpool);
662 svn_pool_destroy(subpool);
664 return svn_error_trace(err);
668 svn_fs_fs__with_write_lock(svn_fs_t *fs,
669 svn_error_t *(*body)(void *baton,
674 fs_fs_data_t *ffd = fs->fsap_data;
675 fs_fs_shared_data_t *ffsd = ffd->shared;
677 SVN_MUTEX__WITH_LOCK(ffsd->fs_write_lock,
678 with_some_lock_file(fs, body, baton,
686 /* Run BODY (with BATON and POOL) while the txn-current file
689 with_txn_current_lock(svn_fs_t *fs,
690 svn_error_t *(*body)(void *baton,
695 fs_fs_data_t *ffd = fs->fsap_data;
696 fs_fs_shared_data_t *ffsd = ffd->shared;
698 SVN_MUTEX__WITH_LOCK(ffsd->txn_current_lock,
699 with_some_lock_file(fs, body, baton,
700 path_txn_current_lock(fs, pool),
707 /* A structure used by unlock_proto_rev() and unlock_proto_rev_body(),
709 struct unlock_proto_rev_baton
715 /* Callback used in the implementation of unlock_proto_rev(). */
717 unlock_proto_rev_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool)
719 const struct unlock_proto_rev_baton *b = baton;
720 const char *txn_id = b->txn_id;
721 apr_file_t *lockfile = b->lockcookie;
722 fs_fs_shared_txn_data_t *txn = get_shared_txn(fs, txn_id, FALSE);
723 apr_status_t apr_err;
726 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
727 _("Can't unlock unknown transaction '%s'"),
729 if (!txn->being_written)
730 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
731 _("Can't unlock nonlocked transaction '%s'"),
734 apr_err = apr_file_unlock(lockfile);
736 return svn_error_wrap_apr
738 _("Can't unlock prototype revision lockfile for transaction '%s'"),
740 apr_err = apr_file_close(lockfile);
742 return svn_error_wrap_apr
744 _("Can't close prototype revision lockfile for transaction '%s'"),
747 txn->being_written = FALSE;
752 /* Unlock the prototype revision file for transaction TXN_ID in filesystem
753 FS using cookie LOCKCOOKIE. The original prototype revision file must
754 have been closed _before_ calling this function.
756 Perform temporary allocations in POOL. */
758 unlock_proto_rev(svn_fs_t *fs, const char *txn_id, void *lockcookie,
761 struct unlock_proto_rev_baton b;
764 b.lockcookie = lockcookie;
765 return with_txnlist_lock(fs, unlock_proto_rev_body, &b, pool);
768 /* Same as unlock_proto_rev(), but requires that the transaction list
769 lock is already held. */
771 unlock_proto_rev_list_locked(svn_fs_t *fs, const char *txn_id,
775 struct unlock_proto_rev_baton b;
778 b.lockcookie = lockcookie;
779 return unlock_proto_rev_body(fs, &b, pool);
782 /* A structure used by get_writable_proto_rev() and
783 get_writable_proto_rev_body(), which see. */
784 struct get_writable_proto_rev_baton
791 /* Callback used in the implementation of get_writable_proto_rev(). */
793 get_writable_proto_rev_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool)
795 const struct get_writable_proto_rev_baton *b = baton;
796 apr_file_t **file = b->file;
797 void **lockcookie = b->lockcookie;
798 const char *txn_id = b->txn_id;
800 fs_fs_shared_txn_data_t *txn = get_shared_txn(fs, txn_id, TRUE);
802 /* First, ensure that no thread in this process (including this one)
803 is currently writing to this transaction's proto-rev file. */
804 if (txn->being_written)
805 return svn_error_createf(SVN_ERR_FS_REP_BEING_WRITTEN, NULL,
806 _("Cannot write to the prototype revision file "
807 "of transaction '%s' because a previous "
808 "representation is currently being written by "
813 /* We know that no thread in this process is writing to the proto-rev
814 file, and by extension, that no thread in this process is holding a
815 lock on the prototype revision lock file. It is therefore safe
816 for us to attempt to lock this file, to see if any other process
817 is holding a lock. */
820 apr_file_t *lockfile;
821 apr_status_t apr_err;
822 const char *lockfile_path = path_txn_proto_rev_lock(fs, txn_id, pool);
824 /* Open the proto-rev lockfile, creating it if necessary, as it may
825 not exist if the transaction dates from before the lockfiles were
828 ### We'd also like to use something like svn_io_file_lock2(), but
829 that forces us to create a subpool just to be able to unlock
830 the file, which seems a waste. */
831 SVN_ERR(svn_io_file_open(&lockfile, lockfile_path,
832 APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool));
834 apr_err = apr_file_lock(lockfile,
835 APR_FLOCK_EXCLUSIVE | APR_FLOCK_NONBLOCK);
838 svn_error_clear(svn_io_file_close(lockfile, pool));
840 if (APR_STATUS_IS_EAGAIN(apr_err))
841 return svn_error_createf(SVN_ERR_FS_REP_BEING_WRITTEN, NULL,
842 _("Cannot write to the prototype revision "
843 "file of transaction '%s' because a "
844 "previous representation is currently "
845 "being written by another process"),
848 return svn_error_wrap_apr(apr_err,
849 _("Can't get exclusive lock on file '%s'"),
850 svn_dirent_local_style(lockfile_path, pool));
853 *lockcookie = lockfile;
856 /* We've successfully locked the transaction; mark it as such. */
857 txn->being_written = TRUE;
860 /* Now open the prototype revision file and seek to the end. */
861 err = svn_io_file_open(file, path_txn_proto_rev(fs, txn_id, pool),
862 APR_WRITE | APR_BUFFERED, APR_OS_DEFAULT, pool);
864 /* You might expect that we could dispense with the following seek
865 and achieve the same thing by opening the file using APR_APPEND.
866 Unfortunately, APR's buffered file implementation unconditionally
867 places its initial file pointer at the start of the file (even for
868 files opened with APR_APPEND), so we need this seek to reconcile
869 the APR file pointer to the OS file pointer (since we need to be
870 able to read the current file position later). */
873 apr_off_t offset = 0;
874 err = svn_io_file_seek(*file, APR_END, &offset, pool);
879 err = svn_error_compose_create(
881 unlock_proto_rev_list_locked(fs, txn_id, *lockcookie, pool));
886 return svn_error_trace(err);
889 /* Get a handle to the prototype revision file for transaction TXN_ID in
890 filesystem FS, and lock it for writing. Return FILE, a file handle
891 positioned at the end of the file, and LOCKCOOKIE, a cookie that
892 should be passed to unlock_proto_rev() to unlock the file once FILE
895 If the prototype revision file is already locked, return error
896 SVN_ERR_FS_REP_BEING_WRITTEN.
898 Perform all allocations in POOL. */
900 get_writable_proto_rev(apr_file_t **file,
902 svn_fs_t *fs, const char *txn_id,
905 struct get_writable_proto_rev_baton b;
908 b.lockcookie = lockcookie;
911 return with_txnlist_lock(fs, get_writable_proto_rev_body, &b, pool);
914 /* Callback used in the implementation of purge_shared_txn(). */
916 purge_shared_txn_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool)
918 const char *txn_id = baton;
920 free_shared_txn(fs, txn_id);
921 svn_fs_fs__reset_txn_caches(fs);
926 /* Purge the shared data for transaction TXN_ID in filesystem FS.
927 Perform all allocations in POOL. */
929 purge_shared_txn(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool)
931 return with_txnlist_lock(fs, purge_shared_txn_body, txn_id, pool);
936 /* Fetch the current offset of FILE into *OFFSET_P. */
938 get_file_offset(apr_off_t *offset_p, apr_file_t *file, apr_pool_t *pool)
942 /* Note that, for buffered files, one (possibly surprising) side-effect
943 of this call is to flush any unwritten data to disk. */
945 SVN_ERR(svn_io_file_seek(file, APR_CUR, &offset, pool));
952 /* Check that BUF, a nul-terminated buffer of text from file PATH,
953 contains only digits at OFFSET and beyond, raising an error if not.
954 TITLE contains a user-visible description of the file, usually the
957 Uses POOL for temporary allocation. */
959 check_file_buffer_numeric(const char *buf, apr_off_t offset,
960 const char *path, const char *title,
965 for (p = buf + offset; *p; p++)
966 if (!svn_ctype_isdigit(*p))
967 return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL,
968 _("%s file '%s' contains unexpected non-digit '%c' within '%s'"),
969 title, svn_dirent_local_style(path, pool), *p, buf);
974 /* Check that BUF, a nul-terminated buffer of text from format file PATH,
975 contains only digits at OFFSET and beyond, raising an error if not.
977 Uses POOL for temporary allocation. */
979 check_format_file_buffer_numeric(const char *buf, apr_off_t offset,
980 const char *path, apr_pool_t *pool)
982 return check_file_buffer_numeric(buf, offset, path, "Format", pool);
985 /* Return the error SVN_ERR_FS_UNSUPPORTED_FORMAT if FS's format
986 number is not the same as a format number supported by this
989 check_format(int format)
991 /* Blacklist. These formats may be either younger or older than
992 SVN_FS_FS__FORMAT_NUMBER, but we don't support them. */
993 if (format == SVN_FS_FS__PACKED_REVPROP_SQLITE_DEV_FORMAT)
994 return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL,
995 _("Found format '%d', only created by "
996 "unreleased dev builds; see "
997 "http://subversion.apache.org"
998 "/docs/release-notes/1.7#revprop-packing"),
1001 /* We support all formats from 1-current simultaneously */
1002 if (1 <= format && format <= SVN_FS_FS__FORMAT_NUMBER)
1003 return SVN_NO_ERROR;
1005 return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL,
1006 _("Expected FS format between '1' and '%d'; found format '%d'"),
1007 SVN_FS_FS__FORMAT_NUMBER, format);
1010 /* Read the format number and maximum number of files per directory
1011 from PATH and return them in *PFORMAT and *MAX_FILES_PER_DIR
1014 *MAX_FILES_PER_DIR is obtained from the 'layout' format option, and
1015 will be set to zero if a linear scheme should be used.
1017 Use POOL for temporary allocation. */
1018 static svn_error_t *
1019 read_format(int *pformat, int *max_files_per_dir,
1020 const char *path, apr_pool_t *pool)
1023 svn_stream_t *stream;
1024 svn_stringbuf_t *content;
1025 svn_stringbuf_t *buf;
1026 svn_boolean_t eos = FALSE;
1028 err = svn_stringbuf_from_file2(&content, path, pool);
1029 if (err && APR_STATUS_IS_ENOENT(err->apr_err))
1031 /* Treat an absent format file as format 1. Do not try to
1032 create the format file on the fly, because the repository
1033 might be read-only for us, or this might be a read-only
1034 operation, and the spirit of FSFS is to make no changes
1035 whatseover in read-only operations. See thread starting at
1036 http://subversion.tigris.org/servlets/ReadMsg?list=dev&msgNo=97600
1038 svn_error_clear(err);
1040 *max_files_per_dir = 0;
1042 return SVN_NO_ERROR;
1046 stream = svn_stream_from_stringbuf(content, pool);
1047 SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, pool));
1048 if (buf->len == 0 && eos)
1050 /* Return a more useful error message. */
1051 return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL,
1052 _("Can't read first line of format file '%s'"),
1053 svn_dirent_local_style(path, pool));
1056 /* Check that the first line contains only digits. */
1057 SVN_ERR(check_format_file_buffer_numeric(buf->data, 0, path, pool));
1058 SVN_ERR(svn_cstring_atoi(pformat, buf->data));
1060 /* Check that we support this format at all */
1061 SVN_ERR(check_format(*pformat));
1063 /* Set the default values for anything that can be set via an option. */
1064 *max_files_per_dir = 0;
1066 /* Read any options. */
1069 SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, pool));
1073 if (*pformat >= SVN_FS_FS__MIN_LAYOUT_FORMAT_OPTION_FORMAT &&
1074 strncmp(buf->data, "layout ", 7) == 0)
1076 if (strcmp(buf->data + 7, "linear") == 0)
1078 *max_files_per_dir = 0;
1082 if (strncmp(buf->data + 7, "sharded ", 8) == 0)
1084 /* Check that the argument is numeric. */
1085 SVN_ERR(check_format_file_buffer_numeric(buf->data, 15, path, pool));
1086 SVN_ERR(svn_cstring_atoi(max_files_per_dir, buf->data + 15));
1091 return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL,
1092 _("'%s' contains invalid filesystem format option '%s'"),
1093 svn_dirent_local_style(path, pool), buf->data);
1096 return SVN_NO_ERROR;
1099 /* Write the format number and maximum number of files per directory
1100 to a new format file in PATH, possibly expecting to overwrite a
1101 previously existing file.
1103 Use POOL for temporary allocation. */
1104 static svn_error_t *
1105 write_format(const char *path, int format, int max_files_per_dir,
1106 svn_boolean_t overwrite, apr_pool_t *pool)
1108 svn_stringbuf_t *sb;
1110 SVN_ERR_ASSERT(1 <= format && format <= SVN_FS_FS__FORMAT_NUMBER);
1112 sb = svn_stringbuf_createf(pool, "%d\n", format);
1114 if (format >= SVN_FS_FS__MIN_LAYOUT_FORMAT_OPTION_FORMAT)
1116 if (max_files_per_dir)
1117 svn_stringbuf_appendcstr(sb, apr_psprintf(pool, "layout sharded %d\n",
1118 max_files_per_dir));
1120 svn_stringbuf_appendcstr(sb, "layout linear\n");
1123 /* svn_io_write_version_file() does a load of magic to allow it to
1124 replace version files that already exist. We only need to do
1125 that when we're allowed to overwrite an existing file. */
1128 /* Create the file */
1129 SVN_ERR(svn_io_file_create(path, sb->data, pool));
1133 const char *path_tmp;
1135 SVN_ERR(svn_io_write_unique(&path_tmp,
1136 svn_dirent_dirname(path, pool),
1138 svn_io_file_del_none, pool));
1140 /* rename the temp file as the real destination */
1141 SVN_ERR(svn_io_file_rename(path_tmp, path, pool));
1144 /* And set the perms to make it read only */
1145 return svn_io_set_file_read_only(path, FALSE, pool);
1149 svn_fs_fs__fs_supports_mergeinfo(svn_fs_t *fs)
1151 fs_fs_data_t *ffd = fs->fsap_data;
1152 return ffd->format >= SVN_FS_FS__MIN_MERGEINFO_FORMAT;
1155 /* Read the configuration information of the file system at FS_PATH
1156 * and set the respective values in FFD. Use POOL for allocations.
1158 static svn_error_t *
1159 read_config(fs_fs_data_t *ffd,
1160 const char *fs_path,
1163 SVN_ERR(svn_config_read3(&ffd->config,
1164 svn_dirent_join(fs_path, PATH_CONFIG, pool),
1165 FALSE, FALSE, FALSE, pool));
1167 /* Initialize ffd->rep_sharing_allowed. */
1168 if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT)
1169 SVN_ERR(svn_config_get_bool(ffd->config, &ffd->rep_sharing_allowed,
1170 CONFIG_SECTION_REP_SHARING,
1171 CONFIG_OPTION_ENABLE_REP_SHARING, TRUE));
1173 ffd->rep_sharing_allowed = FALSE;
1175 /* Initialize deltification settings in ffd. */
1176 if (ffd->format >= SVN_FS_FS__MIN_DELTIFICATION_FORMAT)
1178 SVN_ERR(svn_config_get_bool(ffd->config, &ffd->deltify_directories,
1179 CONFIG_SECTION_DELTIFICATION,
1180 CONFIG_OPTION_ENABLE_DIR_DELTIFICATION,
1182 SVN_ERR(svn_config_get_bool(ffd->config, &ffd->deltify_properties,
1183 CONFIG_SECTION_DELTIFICATION,
1184 CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION,
1186 SVN_ERR(svn_config_get_int64(ffd->config, &ffd->max_deltification_walk,
1187 CONFIG_SECTION_DELTIFICATION,
1188 CONFIG_OPTION_MAX_DELTIFICATION_WALK,
1189 SVN_FS_FS_MAX_DELTIFICATION_WALK));
1190 SVN_ERR(svn_config_get_int64(ffd->config, &ffd->max_linear_deltification,
1191 CONFIG_SECTION_DELTIFICATION,
1192 CONFIG_OPTION_MAX_LINEAR_DELTIFICATION,
1193 SVN_FS_FS_MAX_LINEAR_DELTIFICATION));
1197 ffd->deltify_directories = FALSE;
1198 ffd->deltify_properties = FALSE;
1199 ffd->max_deltification_walk = SVN_FS_FS_MAX_DELTIFICATION_WALK;
1200 ffd->max_linear_deltification = SVN_FS_FS_MAX_LINEAR_DELTIFICATION;
1203 /* Initialize revprop packing settings in ffd. */
1204 if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
1206 SVN_ERR(svn_config_get_bool(ffd->config, &ffd->compress_packed_revprops,
1207 CONFIG_SECTION_PACKED_REVPROPS,
1208 CONFIG_OPTION_COMPRESS_PACKED_REVPROPS,
1210 SVN_ERR(svn_config_get_int64(ffd->config, &ffd->revprop_pack_size,
1211 CONFIG_SECTION_PACKED_REVPROPS,
1212 CONFIG_OPTION_REVPROP_PACK_SIZE,
1213 ffd->compress_packed_revprops
1217 ffd->revprop_pack_size *= 1024;
1221 ffd->revprop_pack_size = 0x10000;
1222 ffd->compress_packed_revprops = FALSE;
1225 return SVN_NO_ERROR;
1228 static svn_error_t *
1229 write_config(svn_fs_t *fs,
1232 #define NL APR_EOL_STR
1233 static const char * const fsfs_conf_contents =
1234 "### This file controls the configuration of the FSFS filesystem." NL
1236 "[" SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS "]" NL
1237 "### These options name memcached servers used to cache internal FSFS" NL
1238 "### data. See http://www.danga.com/memcached/ for more information on" NL
1239 "### memcached. To use memcached with FSFS, run one or more memcached" NL
1240 "### servers, and specify each of them as an option like so:" NL
1241 "# first-server = 127.0.0.1:11211" NL
1242 "# remote-memcached = mymemcached.corp.example.com:11212" NL
1243 "### The option name is ignored; the value is of the form HOST:PORT." NL
1244 "### memcached servers can be shared between multiple repositories;" NL
1245 "### however, if you do this, you *must* ensure that repositories have" NL
1246 "### distinct UUIDs and paths, or else cached data from one repository" NL
1247 "### might be used by another accidentally. Note also that memcached has" NL
1248 "### no authentication for reads or writes, so you must ensure that your" NL
1249 "### memcached servers are only accessible by trusted users." NL
1251 "[" CONFIG_SECTION_CACHES "]" NL
1252 "### When a cache-related error occurs, normally Subversion ignores it" NL
1253 "### and continues, logging an error if the server is appropriately" NL
1254 "### configured (and ignoring it with file:// access). To make" NL
1255 "### Subversion never ignore cache errors, uncomment this line." NL
1256 "# " CONFIG_OPTION_FAIL_STOP " = true" NL
1258 "[" CONFIG_SECTION_REP_SHARING "]" NL
1259 "### To conserve space, the filesystem can optionally avoid storing" NL
1260 "### duplicate representations. This comes at a slight cost in" NL
1261 "### performance, as maintaining a database of shared representations can" NL
1262 "### increase commit times. The space savings are dependent upon the size" NL
1263 "### of the repository, the number of objects it contains and the amount of" NL
1264 "### duplication between them, usually a function of the branching and" NL
1265 "### merging process." NL
1267 "### The following parameter enables rep-sharing in the repository. It can" NL
1268 "### be switched on and off at will, but for best space-saving results" NL
1269 "### should be enabled consistently over the life of the repository." NL
1270 "### 'svnadmin verify' will check the rep-cache regardless of this setting." NL
1271 "### rep-sharing is enabled by default." NL
1272 "# " CONFIG_OPTION_ENABLE_REP_SHARING " = true" NL
1274 "[" CONFIG_SECTION_DELTIFICATION "]" NL
1275 "### To conserve space, the filesystem stores data as differences against" NL
1276 "### existing representations. This comes at a slight cost in performance," NL
1277 "### as calculating differences can increase commit times. Reading data" NL
1278 "### will also create higher CPU load and the data will be fragmented." NL
1279 "### Since deltification tends to save significant amounts of disk space," NL
1280 "### the overall I/O load can actually be lower." NL
1282 "### The options in this section allow for tuning the deltification" NL
1283 "### strategy. Their effects on data size and server performance may vary" NL
1284 "### from one repository to another. Versions prior to 1.8 will ignore" NL
1285 "### this section." NL
1287 "### The following parameter enables deltification for directories. It can" NL
1288 "### be switched on and off at will, but for best space-saving results" NL
1289 "### should be enabled consistently over the life of the repository." NL
1290 "### Repositories containing large directories will benefit greatly." NL
1291 "### In rarely read repositories, the I/O overhead may be significant as" NL
1292 "### cache hit rates will most likely be low" NL
1293 "### directory deltification is disabled by default." NL
1294 "# " CONFIG_OPTION_ENABLE_DIR_DELTIFICATION " = false" NL
1296 "### The following parameter enables deltification for properties on files" NL
1297 "### and directories. Overall, this is a minor tuning option but can save" NL
1298 "### some disk space if you merge frequently or frequently change node" NL
1299 "### properties. You should not activate this if rep-sharing has been" NL
1300 "### disabled because this may result in a net increase in repository size." NL
1301 "### property deltification is disabled by default." NL
1302 "# " CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION " = false" NL
1304 "### During commit, the server may need to walk the whole change history of" NL
1305 "### of a given node to find a suitable deltification base. This linear" NL
1306 "### process can impact commit times, svnadmin load and similar operations." NL
1307 "### This setting limits the depth of the deltification history. If the" NL
1308 "### threshold has been reached, the node will be stored as fulltext and a" NL
1309 "### new deltification history begins." NL
1310 "### Note, this is unrelated to svn log." NL
1311 "### Very large values rarely provide significant additional savings but" NL
1312 "### can impact performance greatly - in particular if directory" NL
1313 "### deltification has been activated. Very small values may be useful in" NL
1314 "### repositories that are dominated by large, changing binaries." NL
1315 "### Should be a power of two minus 1. A value of 0 will effectively" NL
1316 "### disable deltification." NL
1317 "### For 1.8, the default value is 1023; earlier versions have no limit." NL
1318 "# " CONFIG_OPTION_MAX_DELTIFICATION_WALK " = 1023" NL
1320 "### The skip-delta scheme used by FSFS tends to repeatably store redundant" NL
1321 "### delta information where a simple delta against the latest version is" NL
1322 "### often smaller. By default, 1.8+ will therefore use skip deltas only" NL
1323 "### after the linear chain of deltas has grown beyond the threshold" NL
1324 "### specified by this setting." NL
1325 "### Values up to 64 can result in some reduction in repository size for" NL
1326 "### the cost of quickly increasing I/O and CPU costs. Similarly, smaller" NL
1327 "### numbers can reduce those costs at the cost of more disk space. For" NL
1328 "### rarely read repositories or those containing larger binaries, this may" NL
1329 "### present a better trade-off." NL
1330 "### Should be a power of two. A value of 1 or smaller will cause the" NL
1331 "### exclusive use of skip-deltas (as in pre-1.8)." NL
1332 "### For 1.8, the default value is 16; earlier versions use 1." NL
1333 "# " CONFIG_OPTION_MAX_LINEAR_DELTIFICATION " = 16" NL
1335 "[" CONFIG_SECTION_PACKED_REVPROPS "]" NL
1336 "### This parameter controls the size (in kBytes) of packed revprop files." NL
1337 "### Revprops of consecutive revisions will be concatenated into a single" NL
1338 "### file up to but not exceeding the threshold given here. However, each" NL
1339 "### pack file may be much smaller and revprops of a single revision may be" NL
1340 "### much larger than the limit set here. The threshold will be applied" NL
1341 "### before optional compression takes place." NL
1342 "### Large values will reduce disk space usage at the expense of increased" NL
1343 "### latency and CPU usage reading and changing individual revprops. They" NL
1344 "### become an advantage when revprop caching has been enabled because a" NL
1345 "### lot of data can be read in one go. Values smaller than 4 kByte will" NL
1346 "### not improve latency any further and quickly render revprop packing" NL
1347 "### ineffective." NL
1348 "### revprop-pack-size is 64 kBytes by default for non-compressed revprop" NL
1349 "### pack files and 256 kBytes when compression has been enabled." NL
1350 "# " CONFIG_OPTION_REVPROP_PACK_SIZE " = 64" NL
1352 "### To save disk space, packed revprop files may be compressed. Standard" NL
1353 "### revprops tend to allow for very effective compression. Reading and" NL
1354 "### even more so writing, become significantly more CPU intensive. With" NL
1355 "### revprop caching enabled, the overhead can be offset by reduced I/O" NL
1356 "### unless you often modify revprops after packing." NL
1357 "### Compressing packed revprops is disabled by default." NL
1358 "# " CONFIG_OPTION_COMPRESS_PACKED_REVPROPS " = false" NL
1361 return svn_io_file_create(svn_dirent_join(fs->path, PATH_CONFIG, pool),
1362 fsfs_conf_contents, pool);
1365 static svn_error_t *
1366 read_min_unpacked_rev(svn_revnum_t *min_unpacked_rev,
1374 SVN_ERR(svn_io_file_open(&file, path, APR_READ | APR_BUFFERED,
1375 APR_OS_DEFAULT, pool));
1377 SVN_ERR(svn_io_read_length_line(file, buf, &len, pool));
1378 SVN_ERR(svn_io_file_close(file, pool));
1380 *min_unpacked_rev = SVN_STR_TO_REV(buf);
1381 return SVN_NO_ERROR;
1384 static svn_error_t *
1385 update_min_unpacked_rev(svn_fs_t *fs, apr_pool_t *pool)
1387 fs_fs_data_t *ffd = fs->fsap_data;
1389 SVN_ERR_ASSERT(ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT);
1391 return read_min_unpacked_rev(&ffd->min_unpacked_rev,
1392 path_min_unpacked_rev(fs, pool),
1397 svn_fs_fs__open(svn_fs_t *fs, const char *path, apr_pool_t *pool)
1399 fs_fs_data_t *ffd = fs->fsap_data;
1400 apr_file_t *uuid_file;
1401 int format, max_files_per_dir;
1402 char buf[APR_UUID_FORMATTED_LENGTH + 2];
1405 fs->path = apr_pstrdup(fs->pool, path);
1407 /* Read the FS format number. */
1408 SVN_ERR(read_format(&format, &max_files_per_dir,
1409 path_format(fs, pool), pool));
1411 /* Now we've got a format number no matter what. */
1412 ffd->format = format;
1413 ffd->max_files_per_dir = max_files_per_dir;
1415 /* Read in and cache the repository uuid. */
1416 SVN_ERR(svn_io_file_open(&uuid_file, path_uuid(fs, pool),
1417 APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
1419 limit = sizeof(buf);
1420 SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit, pool));
1421 fs->uuid = apr_pstrdup(fs->pool, buf);
1423 SVN_ERR(svn_io_file_close(uuid_file, pool));
1425 /* Read the min unpacked revision. */
1426 if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
1427 SVN_ERR(update_min_unpacked_rev(fs, pool));
1429 /* Read the configuration file. */
1430 SVN_ERR(read_config(ffd, fs->path, pool));
1432 return get_youngest(&(ffd->youngest_rev_cache), path, pool);
1435 /* Wrapper around svn_io_file_create which ignores EEXIST. */
1436 static svn_error_t *
1437 create_file_ignore_eexist(const char *file,
1438 const char *contents,
1441 svn_error_t *err = svn_io_file_create(file, contents, pool);
1442 if (err && APR_STATUS_IS_EEXIST(err->apr_err))
1444 svn_error_clear(err);
1447 return svn_error_trace(err);
1450 /* forward declarations */
1452 static svn_error_t *
1453 pack_revprops_shard(const char *pack_file_dir,
1454 const char *shard_path,
1456 int max_files_per_dir,
1457 apr_off_t max_pack_size,
1458 int compression_level,
1459 svn_cancel_func_t cancel_func,
1461 apr_pool_t *scratch_pool);
1463 static svn_error_t *
1464 delete_revprops_shard(const char *shard_path,
1466 int max_files_per_dir,
1467 svn_cancel_func_t cancel_func,
1469 apr_pool_t *scratch_pool);
1471 /* In the filesystem FS, pack all revprop shards up to min_unpacked_rev.
1473 * NOTE: Keep the old non-packed shards around until after the format bump.
1474 * Otherwise, re-running upgrade will drop the packed revprop shard but
1475 * have no unpacked data anymore. Call upgrade_cleanup_pack_revprops after
1478 * Use SCRATCH_POOL for temporary allocations.
1480 static svn_error_t *
1481 upgrade_pack_revprops(svn_fs_t *fs,
1482 apr_pool_t *scratch_pool)
1484 fs_fs_data_t *ffd = fs->fsap_data;
1485 const char *revprops_shard_path;
1486 const char *revprops_pack_file_dir;
1488 apr_int64_t first_unpacked_shard
1489 = ffd->min_unpacked_rev / ffd->max_files_per_dir;
1491 apr_pool_t *iterpool = svn_pool_create(scratch_pool);
1492 const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR,
1494 int compression_level = ffd->compress_packed_revprops
1495 ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
1496 : SVN_DELTA_COMPRESSION_LEVEL_NONE;
1498 /* first, pack all revprops shards to match the packed revision shards */
1499 for (shard = 0; shard < first_unpacked_shard; ++shard)
1501 revprops_pack_file_dir = svn_dirent_join(revsprops_dir,
1502 apr_psprintf(iterpool,
1503 "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
1506 revprops_shard_path = svn_dirent_join(revsprops_dir,
1507 apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
1510 SVN_ERR(pack_revprops_shard(revprops_pack_file_dir, revprops_shard_path,
1511 shard, ffd->max_files_per_dir,
1512 (int)(0.9 * ffd->revprop_pack_size),
1514 NULL, NULL, iterpool));
1515 svn_pool_clear(iterpool);
1518 svn_pool_destroy(iterpool);
1520 return SVN_NO_ERROR;
1523 /* In the filesystem FS, remove all non-packed revprop shards up to
1524 * min_unpacked_rev. Use SCRATCH_POOL for temporary allocations.
1525 * See upgrade_pack_revprops for more info.
1527 static svn_error_t *
1528 upgrade_cleanup_pack_revprops(svn_fs_t *fs,
1529 apr_pool_t *scratch_pool)
1531 fs_fs_data_t *ffd = fs->fsap_data;
1532 const char *revprops_shard_path;
1534 apr_int64_t first_unpacked_shard
1535 = ffd->min_unpacked_rev / ffd->max_files_per_dir;
1537 apr_pool_t *iterpool = svn_pool_create(scratch_pool);
1538 const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR,
1541 /* delete the non-packed revprops shards afterwards */
1542 for (shard = 0; shard < first_unpacked_shard; ++shard)
1544 revprops_shard_path = svn_dirent_join(revsprops_dir,
1545 apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
1547 SVN_ERR(delete_revprops_shard(revprops_shard_path,
1548 shard, ffd->max_files_per_dir,
1549 NULL, NULL, iterpool));
1550 svn_pool_clear(iterpool);
1553 svn_pool_destroy(iterpool);
1555 return SVN_NO_ERROR;
1558 static svn_error_t *
1559 upgrade_body(void *baton, apr_pool_t *pool)
1561 svn_fs_t *fs = baton;
1562 int format, max_files_per_dir;
1563 const char *format_path = path_format(fs, pool);
1564 svn_node_kind_t kind;
1565 svn_boolean_t needs_revprop_shard_cleanup = FALSE;
1567 /* Read the FS format number and max-files-per-dir setting. */
1568 SVN_ERR(read_format(&format, &max_files_per_dir, format_path, pool));
1570 /* If the config file does not exist, create one. */
1571 SVN_ERR(svn_io_check_path(svn_dirent_join(fs->path, PATH_CONFIG, pool),
1576 SVN_ERR(write_config(fs, pool));
1581 return svn_error_createf(SVN_ERR_FS_GENERAL, NULL,
1582 _("'%s' is not a regular file."
1583 " Please move it out of "
1584 "the way and try again"),
1585 svn_dirent_join(fs->path, PATH_CONFIG, pool));
1588 /* If we're already up-to-date, there's nothing else to be done here. */
1589 if (format == SVN_FS_FS__FORMAT_NUMBER)
1590 return SVN_NO_ERROR;
1592 /* If our filesystem predates the existance of the 'txn-current
1593 file', make that file and its corresponding lock file. */
1594 if (format < SVN_FS_FS__MIN_TXN_CURRENT_FORMAT)
1596 SVN_ERR(create_file_ignore_eexist(path_txn_current(fs, pool), "0\n",
1598 SVN_ERR(create_file_ignore_eexist(path_txn_current_lock(fs, pool), "",
1602 /* If our filesystem predates the existance of the 'txn-protorevs'
1603 dir, make that directory. */
1604 if (format < SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT)
1606 /* We don't use path_txn_proto_rev() here because it expects
1607 we've already bumped our format. */
1608 SVN_ERR(svn_io_make_dir_recursively(
1609 svn_dirent_join(fs->path, PATH_TXN_PROTOS_DIR, pool), pool));
1612 /* If our filesystem is new enough, write the min unpacked rev file. */
1613 if (format < SVN_FS_FS__MIN_PACKED_FORMAT)
1614 SVN_ERR(svn_io_file_create(path_min_unpacked_rev(fs, pool), "0\n", pool));
1616 /* If the file system supports revision packing but not revprop packing
1617 *and* the FS has been sharded, pack the revprops up to the point that
1618 revision data has been packed. However, keep the non-packed revprop
1619 files around until after the format bump */
1620 if ( format >= SVN_FS_FS__MIN_PACKED_FORMAT
1621 && format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT
1622 && max_files_per_dir > 0)
1624 needs_revprop_shard_cleanup = TRUE;
1625 SVN_ERR(upgrade_pack_revprops(fs, pool));
1628 /* Bump the format file. */
1629 SVN_ERR(write_format(format_path, SVN_FS_FS__FORMAT_NUMBER,
1630 max_files_per_dir, TRUE, pool));
1632 /* Now, it is safe to remove the redundant revprop files. */
1633 if (needs_revprop_shard_cleanup)
1634 SVN_ERR(upgrade_cleanup_pack_revprops(fs, pool));
1637 return SVN_NO_ERROR;
1642 svn_fs_fs__upgrade(svn_fs_t *fs, apr_pool_t *pool)
1644 return svn_fs_fs__with_write_lock(fs, upgrade_body, (void *)fs, pool);
1648 /* Functions for dealing with recoverable errors on mutable files
1650 * Revprops, current, and txn-current files are mutable; that is, they
1651 * change as part of normal fsfs operation, in constrat to revs files, or
1652 * the format file, which are written once at create (or upgrade) time.
1653 * When more than one host writes to the same repository, we will
1654 * sometimes see these recoverable errors when accesssing these files.
1656 * These errors all relate to NFS, and thus we only use this retry code if
1657 * ESTALE is defined.
1661 * In NFS v3 and under, the server doesn't track opened files. If you
1662 * unlink(2) or rename(2) a file held open by another process *on the
1663 * same host*, that host's kernel typically renames the file to
1664 * .nfsXXXX and automatically deletes that when it's no longer open,
1665 * but this behavior is not required.
1667 * For obvious reasons, this does not work *across hosts*. No one
1668 * knows about the opened file; not the server, and not the deleting
1669 * client. So the file vanishes, and the reader gets stale NFS file
1674 * Some client implementations (at least the 2.6.18.5 kernel that ships
1675 * with Ubuntu Dapper) sometimes give spurious ENOENT (only on open) or
1676 * even EIO errors when trying to read these files that have been renamed
1677 * over on some other host.
1681 * Try open and read of such files in try_stringbuf_from_file(). Call
1682 * this function within a loop of RECOVERABLE_RETRY_COUNT iterations
1683 * (though, realistically, the second try will succeed).
1686 #define RECOVERABLE_RETRY_COUNT 10
1688 /* Read the file at PATH and return its content in *CONTENT. *CONTENT will
1689 * not be modified unless the whole file was read successfully.
1691 * ESTALE, EIO and ENOENT will not cause this function to return an error
1692 * unless LAST_ATTEMPT has been set. If MISSING is not NULL, indicate
1693 * missing files (ENOENT) there.
1695 * Use POOL for allocations.
1697 static svn_error_t *
1698 try_stringbuf_from_file(svn_stringbuf_t **content,
1699 svn_boolean_t *missing,
1701 svn_boolean_t last_attempt,
1704 svn_error_t *err = svn_stringbuf_from_file2(content, path, pool);
1712 if (APR_STATUS_IS_ENOENT(err->apr_err))
1716 svn_error_clear(err);
1719 return SVN_NO_ERROR;
1723 else if (APR_TO_OS_ERROR(err->apr_err) == ESTALE
1724 || APR_TO_OS_ERROR(err->apr_err) == EIO)
1728 svn_error_clear(err);
1729 return SVN_NO_ERROR;
1735 return svn_error_trace(err);
1738 /* Read the 'current' file FNAME and store the contents in *BUF.
1739 Allocations are performed in POOL. */
1740 static svn_error_t *
1741 read_content(svn_stringbuf_t **content, const char *fname, apr_pool_t *pool)
1746 for (i = 0; !*content && (i < RECOVERABLE_RETRY_COUNT); ++i)
1747 SVN_ERR(try_stringbuf_from_file(content, NULL,
1748 fname, i + 1 < RECOVERABLE_RETRY_COUNT,
1752 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
1753 _("Can't read '%s'"),
1754 svn_dirent_local_style(fname, pool));
1756 return SVN_NO_ERROR;
1759 /* Find the youngest revision in a repository at path FS_PATH and
1760 return it in *YOUNGEST_P. Perform temporary allocations in
1762 static svn_error_t *
1763 get_youngest(svn_revnum_t *youngest_p,
1764 const char *fs_path,
1767 svn_stringbuf_t *buf;
1768 SVN_ERR(read_content(&buf, svn_dirent_join(fs_path, PATH_CURRENT, pool),
1771 *youngest_p = SVN_STR_TO_REV(buf->data);
1773 return SVN_NO_ERROR;
1778 svn_fs_fs__youngest_rev(svn_revnum_t *youngest_p,
1782 fs_fs_data_t *ffd = fs->fsap_data;
1784 SVN_ERR(get_youngest(youngest_p, fs->path, pool));
1785 ffd->youngest_rev_cache = *youngest_p;
1787 return SVN_NO_ERROR;
1790 /* Given a revision file FILE that has been pre-positioned at the
1791 beginning of a Node-Rev header block, read in that header block and
1792 store it in the apr_hash_t HEADERS. All allocations will be from
1794 static svn_error_t * read_header_block(apr_hash_t **headers,
1795 svn_stream_t *stream,
1798 *headers = apr_hash_make(pool);
1802 svn_stringbuf_t *header_str;
1803 const char *name, *value;
1807 SVN_ERR(svn_stream_readline(stream, &header_str, "\n", &eof, pool));
1809 if (eof || header_str->len == 0)
1810 break; /* end of header block */
1812 while (header_str->data[i] != ':')
1814 if (header_str->data[i] == '\0')
1815 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
1816 _("Found malformed header '%s' in "
1822 /* Create a 'name' string and point to it. */
1823 header_str->data[i] = '\0';
1824 name = header_str->data;
1826 /* Skip over the NULL byte and the space following it. */
1829 if (i > header_str->len)
1831 /* Restore the original line for the error. */
1833 header_str->data[i] = ':';
1834 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
1835 _("Found malformed header '%s' in "
1840 value = header_str->data + i;
1842 /* header_str is safely in our pool, so we can use bits of it as
1844 svn_hash_sets(*headers, name, value);
1847 return SVN_NO_ERROR;
1850 /* Return SVN_ERR_FS_NO_SUCH_REVISION if the given revision is newer
1851 than the current youngest revision or is simply not a valid
1852 revision number, else return success.
1854 FSFS is based around the concept that commits only take effect when
1855 the number in "current" is bumped. Thus if there happens to be a rev
1856 or revprops file installed for a revision higher than the one recorded
1857 in "current" (because a commit failed between installing the rev file
1858 and bumping "current", or because an administrator rolled back the
1859 repository by resetting "current" without deleting rev files, etc), it
1860 ought to be completely ignored. This function provides the check
1861 by which callers can make that decision. */
1862 static svn_error_t *
1863 ensure_revision_exists(svn_fs_t *fs,
1867 fs_fs_data_t *ffd = fs->fsap_data;
1869 if (! SVN_IS_VALID_REVNUM(rev))
1870 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
1871 _("Invalid revision number '%ld'"), rev);
1874 /* Did the revision exist the last time we checked the current
1876 if (rev <= ffd->youngest_rev_cache)
1877 return SVN_NO_ERROR;
1879 SVN_ERR(get_youngest(&(ffd->youngest_rev_cache), fs->path, pool));
1882 if (rev <= ffd->youngest_rev_cache)
1883 return SVN_NO_ERROR;
1885 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
1886 _("No such revision %ld"), rev);
1890 svn_fs_fs__revision_exists(svn_revnum_t rev,
1894 /* Different order of parameters. */
1895 SVN_ERR(ensure_revision_exists(fs, rev, pool));
1896 return SVN_NO_ERROR;
1899 /* Open the correct revision file for REV. If the filesystem FS has
1900 been packed, *FILE will be set to the packed file; otherwise, set *FILE
1901 to the revision file for REV. Return SVN_ERR_FS_NO_SUCH_REVISION if the
1904 TODO: Consider returning an indication of whether this is a packed rev
1905 file, so the caller need not rely on is_packed_rev() which in turn
1906 relies on the cached FFD->min_unpacked_rev value not having changed
1907 since the rev file was opened.
1909 Use POOL for allocations. */
1910 static svn_error_t *
1911 open_pack_or_rev_file(apr_file_t **file,
1916 fs_fs_data_t *ffd = fs->fsap_data;
1919 svn_boolean_t retry = FALSE;
1923 err = svn_fs_fs__path_rev_absolute(&path, fs, rev, pool);
1925 /* open the revision file in buffered r/o mode */
1927 err = svn_io_file_open(file, path,
1928 APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool);
1930 if (err && APR_STATUS_IS_ENOENT(err->apr_err))
1932 if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
1934 /* Could not open the file. This may happen if the
1935 * file once existed but got packed later. */
1936 svn_error_clear(err);
1938 /* if that was our 2nd attempt, leave it at that. */
1940 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
1941 _("No such revision %ld"), rev);
1943 /* We failed for the first time. Refresh cache & retry. */
1944 SVN_ERR(update_min_unpacked_rev(fs, pool));
1950 svn_error_clear(err);
1951 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
1952 _("No such revision %ld"), rev);
1962 return svn_error_trace(err);
1965 /* Reads a line from STREAM and converts it to a 64 bit integer to be
1966 * returned in *RESULT. If we encounter eof, set *HIT_EOF and leave
1967 * *RESULT unchanged. If HIT_EOF is NULL, EOF causes an "corrupt FS"
1969 * SCRATCH_POOL is used for temporary allocations.
1971 static svn_error_t *
1972 read_number_from_stream(apr_int64_t *result,
1973 svn_boolean_t *hit_eof,
1974 svn_stream_t *stream,
1975 apr_pool_t *scratch_pool)
1977 svn_stringbuf_t *sb;
1981 SVN_ERR(svn_stream_readline(stream, &sb, "\n", &eof, scratch_pool));
1986 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Unexpected EOF"));
1990 err = svn_cstring_atoi64(result, sb->data);
1992 return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
1993 _("Number '%s' invalid or too large"),
1997 return SVN_NO_ERROR;
2000 /* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file.
2001 Use POOL for temporary allocations. */
2002 static svn_error_t *
2003 get_packed_offset(apr_off_t *rev_offset,
2008 fs_fs_data_t *ffd = fs->fsap_data;
2009 svn_stream_t *manifest_stream;
2010 svn_boolean_t is_cached;
2012 apr_int64_t shard_pos;
2013 apr_array_header_t *manifest;
2014 apr_pool_t *iterpool;
2016 shard = rev / ffd->max_files_per_dir;
2018 /* position of the shard within the manifest */
2019 shard_pos = rev % ffd->max_files_per_dir;
2021 /* fetch exactly that element into *rev_offset, if the manifest is found
2023 SVN_ERR(svn_cache__get_partial((void **) rev_offset, &is_cached,
2024 ffd->packed_offset_cache, &shard,
2025 svn_fs_fs__get_sharded_offset, &shard_pos,
2029 return SVN_NO_ERROR;
2031 /* Open the manifest file. */
2032 SVN_ERR(svn_stream_open_readonly(&manifest_stream,
2033 path_rev_packed(fs, rev, PATH_MANIFEST,
2037 /* While we're here, let's just read the entire manifest file into an array,
2038 so we can cache the entire thing. */
2039 iterpool = svn_pool_create(pool);
2040 manifest = apr_array_make(pool, ffd->max_files_per_dir, sizeof(apr_off_t));
2046 svn_pool_clear(iterpool);
2047 SVN_ERR(read_number_from_stream(&val, &eof, manifest_stream, iterpool));
2051 APR_ARRAY_PUSH(manifest, apr_off_t) = (apr_off_t)val;
2053 svn_pool_destroy(iterpool);
2055 *rev_offset = APR_ARRAY_IDX(manifest, rev % ffd->max_files_per_dir,
2058 /* Close up shop and cache the array. */
2059 SVN_ERR(svn_stream_close(manifest_stream));
2060 return svn_cache__set(ffd->packed_offset_cache, &shard, manifest, pool);
2063 /* Open the revision file for revision REV in filesystem FS and store
2064 the newly opened file in FILE. Seek to location OFFSET before
2065 returning. Perform temporary allocations in POOL. */
2066 static svn_error_t *
2067 open_and_seek_revision(apr_file_t **file,
2073 apr_file_t *rev_file;
2075 SVN_ERR(ensure_revision_exists(fs, rev, pool));
2077 SVN_ERR(open_pack_or_rev_file(&rev_file, fs, rev, pool));
2079 if (is_packed_rev(fs, rev))
2081 apr_off_t rev_offset;
2083 SVN_ERR(get_packed_offset(&rev_offset, fs, rev, pool));
2084 offset += rev_offset;
2087 SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool));
2091 return SVN_NO_ERROR;
2094 /* Open the representation for a node-revision in transaction TXN_ID
2095 in filesystem FS and store the newly opened file in FILE. Seek to
2096 location OFFSET before returning. Perform temporary allocations in
2097 POOL. Only appropriate for file contents, nor props or directory
2099 static svn_error_t *
2100 open_and_seek_transaction(apr_file_t **file,
2103 representation_t *rep,
2106 apr_file_t *rev_file;
2109 SVN_ERR(svn_io_file_open(&rev_file, path_txn_proto_rev(fs, txn_id, pool),
2110 APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
2112 offset = rep->offset;
2113 SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool));
2117 return SVN_NO_ERROR;
2120 /* Given a node-id ID, and a representation REP in filesystem FS, open
2121 the correct file and seek to the correction location. Store this
2122 file in *FILE_P. Perform any allocations in POOL. */
2123 static svn_error_t *
2124 open_and_seek_representation(apr_file_t **file_p,
2126 representation_t *rep,
2130 return open_and_seek_revision(file_p, fs, rep->revision, rep->offset,
2133 return open_and_seek_transaction(file_p, fs, rep->txn_id, rep, pool);
2136 /* Parse the description of a representation from STRING and store it
2137 into *REP_P. If the representation is mutable (the revision is
2138 given as -1), then use TXN_ID for the representation's txn_id
2139 field. If MUTABLE_REP_TRUNCATED is true, then this representation
2140 is for property or directory contents, and no information will be
2141 expected except the "-1" revision number for a mutable
2142 representation. Allocate *REP_P in POOL. */
2143 static svn_error_t *
2144 read_rep_offsets_body(representation_t **rep_p,
2147 svn_boolean_t mutable_rep_truncated,
2150 representation_t *rep;
2154 rep = apr_pcalloc(pool, sizeof(*rep));
2157 str = svn_cstring_tokenize(" ", &string);
2159 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
2160 _("Malformed text representation offset line in node-rev"));
2163 rep->revision = SVN_STR_TO_REV(str);
2164 if (rep->revision == SVN_INVALID_REVNUM)
2166 rep->txn_id = txn_id;
2167 if (mutable_rep_truncated)
2168 return SVN_NO_ERROR;
2171 str = svn_cstring_tokenize(" ", &string);
2173 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
2174 _("Malformed text representation offset line in node-rev"));
2176 SVN_ERR(svn_cstring_atoi64(&val, str));
2177 rep->offset = (apr_off_t)val;
2179 str = svn_cstring_tokenize(" ", &string);
2181 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
2182 _("Malformed text representation offset line in node-rev"));
2184 SVN_ERR(svn_cstring_atoi64(&val, str));
2185 rep->size = (svn_filesize_t)val;
2187 str = svn_cstring_tokenize(" ", &string);
2189 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
2190 _("Malformed text representation offset line in node-rev"));
2192 SVN_ERR(svn_cstring_atoi64(&val, str));
2193 rep->expanded_size = (svn_filesize_t)val;
2195 /* Read in the MD5 hash. */
2196 str = svn_cstring_tokenize(" ", &string);
2197 if ((str == NULL) || (strlen(str) != (APR_MD5_DIGESTSIZE * 2)))
2198 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
2199 _("Malformed text representation offset line in node-rev"));
2201 SVN_ERR(svn_checksum_parse_hex(&rep->md5_checksum, svn_checksum_md5, str,
2204 /* The remaining fields are only used for formats >= 4, so check that. */
2205 str = svn_cstring_tokenize(" ", &string);
2207 return SVN_NO_ERROR;
2209 /* Read the SHA1 hash. */
2210 if (strlen(str) != (APR_SHA1_DIGESTSIZE * 2))
2211 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
2212 _("Malformed text representation offset line in node-rev"));
2214 SVN_ERR(svn_checksum_parse_hex(&rep->sha1_checksum, svn_checksum_sha1, str,
2217 /* Read the uniquifier. */
2218 str = svn_cstring_tokenize(" ", &string);
2220 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
2221 _("Malformed text representation offset line in node-rev"));
2223 rep->uniquifier = apr_pstrdup(pool, str);
2225 return SVN_NO_ERROR;
2228 /* Wrap read_rep_offsets_body(), extracting its TXN_ID from our NODEREV_ID,
2229 and adding an error message. */
2230 static svn_error_t *
2231 read_rep_offsets(representation_t **rep_p,
2233 const svn_fs_id_t *noderev_id,
2234 svn_boolean_t mutable_rep_truncated,
2241 txn_id = svn_fs_fs__id_txn_id(noderev_id);
2245 err = read_rep_offsets_body(rep_p, string, txn_id, mutable_rep_truncated,
2249 const svn_string_t *id_unparsed = svn_fs_fs__id_unparse(noderev_id, pool);
2251 where = apr_psprintf(pool,
2252 _("While reading representation offsets "
2253 "for node-revision '%s':"),
2254 noderev_id ? id_unparsed->data : "(null)");
2256 return svn_error_quick_wrap(err, where);
2259 return SVN_NO_ERROR;
2262 static svn_error_t *
2263 err_dangling_id(svn_fs_t *fs, const svn_fs_id_t *id)
2265 svn_string_t *id_str = svn_fs_fs__id_unparse(id, fs->pool);
2266 return svn_error_createf
2267 (SVN_ERR_FS_ID_NOT_FOUND, 0,
2268 _("Reference to non-existent node '%s' in filesystem '%s'"),
2269 id_str->data, fs->path);
2272 /* Look up the NODEREV_P for ID in FS' node revsion cache. If noderev
2273 * caching has been enabled and the data can be found, IS_CACHED will
2274 * be set to TRUE. The noderev will be allocated from POOL.
2276 * Non-permanent ids (e.g. ids within a TXN) will not be cached.
2278 static svn_error_t *
2279 get_cached_node_revision_body(node_revision_t **noderev_p,
2281 const svn_fs_id_t *id,
2282 svn_boolean_t *is_cached,
2285 fs_fs_data_t *ffd = fs->fsap_data;
2286 if (! ffd->node_revision_cache || svn_fs_fs__id_txn_id(id))
2292 pair_cache_key_t key = { 0 };
2294 key.revision = svn_fs_fs__id_rev(id);
2295 key.second = svn_fs_fs__id_offset(id);
2296 SVN_ERR(svn_cache__get((void **) noderev_p,
2298 ffd->node_revision_cache,
2303 return SVN_NO_ERROR;
2306 /* If noderev caching has been enabled, store the NODEREV_P for the given ID
2307 * in FS' node revsion cache. SCRATCH_POOL is used for temporary allcations.
2309 * Non-permanent ids (e.g. ids within a TXN) will not be cached.
2311 static svn_error_t *
2312 set_cached_node_revision_body(node_revision_t *noderev_p,
2314 const svn_fs_id_t *id,
2315 apr_pool_t *scratch_pool)
2317 fs_fs_data_t *ffd = fs->fsap_data;
2319 if (ffd->node_revision_cache && !svn_fs_fs__id_txn_id(id))
2321 pair_cache_key_t key = { 0 };
2323 key.revision = svn_fs_fs__id_rev(id);
2324 key.second = svn_fs_fs__id_offset(id);
2325 return svn_cache__set(ffd->node_revision_cache,
2331 return SVN_NO_ERROR;
2334 /* Get the node-revision for the node ID in FS.
2335 Set *NODEREV_P to the new node-revision structure, allocated in POOL.
2336 See svn_fs_fs__get_node_revision, which wraps this and adds another
2338 static svn_error_t *
2339 get_node_revision_body(node_revision_t **noderev_p,
2341 const svn_fs_id_t *id,
2344 apr_file_t *revision_file;
2346 svn_boolean_t is_cached = FALSE;
2348 /* First, try a cache lookup. If that succeeds, we are done here. */
2349 SVN_ERR(get_cached_node_revision_body(noderev_p, fs, id, &is_cached, pool));
2351 return SVN_NO_ERROR;
2353 if (svn_fs_fs__id_txn_id(id))
2355 /* This is a transaction node-rev. */
2356 err = svn_io_file_open(&revision_file, path_txn_node_rev(fs, id, pool),
2357 APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool);
2361 /* This is a revision node-rev. */
2362 err = open_and_seek_revision(&revision_file, fs,
2363 svn_fs_fs__id_rev(id),
2364 svn_fs_fs__id_offset(id),
2370 if (APR_STATUS_IS_ENOENT(err->apr_err))
2372 svn_error_clear(err);
2373 return svn_error_trace(err_dangling_id(fs, id));
2376 return svn_error_trace(err);
2379 SVN_ERR(svn_fs_fs__read_noderev(noderev_p,
2380 svn_stream_from_aprfile2(revision_file, FALSE,
2384 /* The noderev is not in cache, yet. Add it, if caching has been enabled. */
2385 return set_cached_node_revision_body(*noderev_p, fs, id, pool);
2389 svn_fs_fs__read_noderev(node_revision_t **noderev_p,
2390 svn_stream_t *stream,
2393 apr_hash_t *headers;
2394 node_revision_t *noderev;
2396 const char *noderev_id;
2398 SVN_ERR(read_header_block(&headers, stream, pool));
2400 noderev = apr_pcalloc(pool, sizeof(*noderev));
2402 /* Read the node-rev id. */
2403 value = svn_hash_gets(headers, HEADER_ID);
2405 /* ### More information: filename/offset coordinates */
2406 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
2407 _("Missing id field in node-rev"));
2409 SVN_ERR(svn_stream_close(stream));
2411 noderev->id = svn_fs_fs__id_parse(value, strlen(value), pool);
2412 noderev_id = value; /* for error messages later */
2414 /* Read the type. */
2415 value = svn_hash_gets(headers, HEADER_TYPE);
2417 if ((value == NULL) ||
2418 (strcmp(value, KIND_FILE) != 0 && strcmp(value, KIND_DIR)))
2419 /* ### s/kind/type/ */
2420 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2421 _("Missing kind field in node-rev '%s'"),
2424 noderev->kind = (strcmp(value, KIND_FILE) == 0) ? svn_node_file
2427 /* Read the 'count' field. */
2428 value = svn_hash_gets(headers, HEADER_COUNT);
2430 SVN_ERR(svn_cstring_atoi(&noderev->predecessor_count, value));
2432 noderev->predecessor_count = 0;
2434 /* Get the properties location. */
2435 value = svn_hash_gets(headers, HEADER_PROPS);
2438 SVN_ERR(read_rep_offsets(&noderev->prop_rep, value,
2439 noderev->id, TRUE, pool));
2442 /* Get the data location. */
2443 value = svn_hash_gets(headers, HEADER_TEXT);
2446 SVN_ERR(read_rep_offsets(&noderev->data_rep, value,
2448 (noderev->kind == svn_node_dir), pool));
2451 /* Get the created path. */
2452 value = svn_hash_gets(headers, HEADER_CPATH);
2455 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2456 _("Missing cpath field in node-rev '%s'"),
2461 noderev->created_path = apr_pstrdup(pool, value);
2464 /* Get the predecessor ID. */
2465 value = svn_hash_gets(headers, HEADER_PRED);
2467 noderev->predecessor_id = svn_fs_fs__id_parse(value, strlen(value),
2470 /* Get the copyroot. */
2471 value = svn_hash_gets(headers, HEADER_COPYROOT);
2474 noderev->copyroot_path = apr_pstrdup(pool, noderev->created_path);
2475 noderev->copyroot_rev = svn_fs_fs__id_rev(noderev->id);
2481 str = svn_cstring_tokenize(" ", &value);
2483 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2484 _("Malformed copyroot line in node-rev '%s'"),
2487 noderev->copyroot_rev = SVN_STR_TO_REV(str);
2490 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2491 _("Malformed copyroot line in node-rev '%s'"),
2493 noderev->copyroot_path = apr_pstrdup(pool, value);
2496 /* Get the copyfrom. */
2497 value = svn_hash_gets(headers, HEADER_COPYFROM);
2500 noderev->copyfrom_path = NULL;
2501 noderev->copyfrom_rev = SVN_INVALID_REVNUM;
2505 char *str = svn_cstring_tokenize(" ", &value);
2507 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2508 _("Malformed copyfrom line in node-rev '%s'"),
2511 noderev->copyfrom_rev = SVN_STR_TO_REV(str);
2514 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2515 _("Malformed copyfrom line in node-rev '%s'"),
2517 noderev->copyfrom_path = apr_pstrdup(pool, value);
2520 /* Get whether this is a fresh txn root. */
2521 value = svn_hash_gets(headers, HEADER_FRESHTXNRT);
2522 noderev->is_fresh_txn_root = (value != NULL);
2524 /* Get the mergeinfo count. */
2525 value = svn_hash_gets(headers, HEADER_MINFO_CNT);
2527 SVN_ERR(svn_cstring_atoi64(&noderev->mergeinfo_count, value));
2529 noderev->mergeinfo_count = 0;
2531 /* Get whether *this* node has mergeinfo. */
2532 value = svn_hash_gets(headers, HEADER_MINFO_HERE);
2533 noderev->has_mergeinfo = (value != NULL);
2535 *noderev_p = noderev;
2537 return SVN_NO_ERROR;
2541 svn_fs_fs__get_node_revision(node_revision_t **noderev_p,
2543 const svn_fs_id_t *id,
2546 svn_error_t *err = get_node_revision_body(noderev_p, fs, id, pool);
2547 if (err && err->apr_err == SVN_ERR_FS_CORRUPT)
2549 svn_string_t *id_string = svn_fs_fs__id_unparse(id, pool);
2550 return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
2551 "Corrupt node-revision '%s'",
2554 return svn_error_trace(err);
2558 /* Return a formatted string, compatible with filesystem format FORMAT,
2559 that represents the location of representation REP. If
2560 MUTABLE_REP_TRUNCATED is given, the rep is for props or dir contents,
2561 and only a "-1" revision number will be given for a mutable rep.
2562 If MAY_BE_CORRUPT is true, guard for NULL when constructing the string.
2563 Perform the allocation from POOL. */
2565 representation_string(representation_t *rep,
2567 svn_boolean_t mutable_rep_truncated,
2568 svn_boolean_t may_be_corrupt,
2571 if (rep->txn_id && mutable_rep_truncated)
2574 #define DISPLAY_MAYBE_NULL_CHECKSUM(checksum) \
2575 ((!may_be_corrupt || (checksum) != NULL) \
2576 ? svn_checksum_to_cstring_display((checksum), pool) \
2579 if (format < SVN_FS_FS__MIN_REP_SHARING_FORMAT || rep->sha1_checksum == NULL)
2580 return apr_psprintf(pool, "%ld %" APR_OFF_T_FMT " %" SVN_FILESIZE_T_FMT
2581 " %" SVN_FILESIZE_T_FMT " %s",
2582 rep->revision, rep->offset, rep->size,
2584 DISPLAY_MAYBE_NULL_CHECKSUM(rep->md5_checksum));
2586 return apr_psprintf(pool, "%ld %" APR_OFF_T_FMT " %" SVN_FILESIZE_T_FMT
2587 " %" SVN_FILESIZE_T_FMT " %s %s %s",
2588 rep->revision, rep->offset, rep->size,
2590 DISPLAY_MAYBE_NULL_CHECKSUM(rep->md5_checksum),
2591 DISPLAY_MAYBE_NULL_CHECKSUM(rep->sha1_checksum),
2594 #undef DISPLAY_MAYBE_NULL_CHECKSUM
2600 svn_fs_fs__write_noderev(svn_stream_t *outfile,
2601 node_revision_t *noderev,
2603 svn_boolean_t include_mergeinfo,
2606 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_ID ": %s\n",
2607 svn_fs_fs__id_unparse(noderev->id,
2610 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_TYPE ": %s\n",
2611 (noderev->kind == svn_node_file) ?
2612 KIND_FILE : KIND_DIR));
2614 if (noderev->predecessor_id)
2615 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_PRED ": %s\n",
2616 svn_fs_fs__id_unparse(noderev->predecessor_id,
2619 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_COUNT ": %d\n",
2620 noderev->predecessor_count));
2622 if (noderev->data_rep)
2623 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_TEXT ": %s\n",
2624 representation_string(noderev->data_rep,
2631 if (noderev->prop_rep)
2632 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_PROPS ": %s\n",
2633 representation_string(noderev->prop_rep, format,
2634 TRUE, FALSE, pool)));
2636 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_CPATH ": %s\n",
2637 noderev->created_path));
2639 if (noderev->copyfrom_path)
2640 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_COPYFROM ": %ld"
2642 noderev->copyfrom_rev,
2643 noderev->copyfrom_path));
2645 if ((noderev->copyroot_rev != svn_fs_fs__id_rev(noderev->id)) ||
2646 (strcmp(noderev->copyroot_path, noderev->created_path) != 0))
2647 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_COPYROOT ": %ld"
2649 noderev->copyroot_rev,
2650 noderev->copyroot_path));
2652 if (noderev->is_fresh_txn_root)
2653 SVN_ERR(svn_stream_puts(outfile, HEADER_FRESHTXNRT ": y\n"));
2655 if (include_mergeinfo)
2657 if (noderev->mergeinfo_count > 0)
2658 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_MINFO_CNT ": %"
2659 APR_INT64_T_FMT "\n",
2660 noderev->mergeinfo_count));
2662 if (noderev->has_mergeinfo)
2663 SVN_ERR(svn_stream_puts(outfile, HEADER_MINFO_HERE ": y\n"));
2666 return svn_stream_puts(outfile, "\n");
2670 svn_fs_fs__put_node_revision(svn_fs_t *fs,
2671 const svn_fs_id_t *id,
2672 node_revision_t *noderev,
2673 svn_boolean_t fresh_txn_root,
2676 fs_fs_data_t *ffd = fs->fsap_data;
2677 apr_file_t *noderev_file;
2678 const char *txn_id = svn_fs_fs__id_txn_id(id);
2680 noderev->is_fresh_txn_root = fresh_txn_root;
2683 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2684 _("Attempted to write to non-transaction '%s'"),
2685 svn_fs_fs__id_unparse(id, pool)->data);
2687 SVN_ERR(svn_io_file_open(&noderev_file, path_txn_node_rev(fs, id, pool),
2688 APR_WRITE | APR_CREATE | APR_TRUNCATE
2689 | APR_BUFFERED, APR_OS_DEFAULT, pool));
2691 SVN_ERR(svn_fs_fs__write_noderev(svn_stream_from_aprfile2(noderev_file, TRUE,
2693 noderev, ffd->format,
2694 svn_fs_fs__fs_supports_mergeinfo(fs),
2697 SVN_ERR(svn_io_file_close(noderev_file, pool));
2699 return SVN_NO_ERROR;
2702 /* For the in-transaction NODEREV within FS, write the sha1->rep mapping
2703 * file in the respective transaction, if rep sharing has been enabled etc.
2704 * Use POOL for temporary allocations.
2706 static svn_error_t *
2707 store_sha1_rep_mapping(svn_fs_t *fs,
2708 node_revision_t *noderev,
2711 fs_fs_data_t *ffd = fs->fsap_data;
2713 /* if rep sharing has been enabled and the noderev has a data rep and
2714 * its SHA-1 is known, store the rep struct under its SHA1. */
2715 if ( ffd->rep_sharing_allowed
2716 && noderev->data_rep
2717 && noderev->data_rep->sha1_checksum)
2719 apr_file_t *rep_file;
2720 const char *file_name = path_txn_sha1(fs,
2721 svn_fs_fs__id_txn_id(noderev->id),
2722 noderev->data_rep->sha1_checksum,
2724 const char *rep_string = representation_string(noderev->data_rep,
2730 SVN_ERR(svn_io_file_open(&rep_file, file_name,
2731 APR_WRITE | APR_CREATE | APR_TRUNCATE
2732 | APR_BUFFERED, APR_OS_DEFAULT, pool));
2734 SVN_ERR(svn_io_file_write_full(rep_file, rep_string,
2735 strlen(rep_string), NULL, pool));
2737 SVN_ERR(svn_io_file_close(rep_file, pool));
2740 return SVN_NO_ERROR;
2744 /* This structure is used to hold the information associated with a
2748 svn_boolean_t is_delta;
2749 svn_boolean_t is_delta_vs_empty;
2751 svn_revnum_t base_revision;
2752 apr_off_t base_offset;
2753 svn_filesize_t base_length;
2756 /* Read the next line from file FILE and parse it as a text
2757 representation entry. Return the parsed entry in *REP_ARGS_P.
2758 Perform all allocations in POOL. */
2759 static svn_error_t *
2760 read_rep_line(struct rep_args **rep_args_p,
2766 struct rep_args *rep_args;
2767 char *str, *last_str = buffer;
2770 limit = sizeof(buffer);
2771 SVN_ERR(svn_io_read_length_line(file, buffer, &limit, pool));
2773 rep_args = apr_pcalloc(pool, sizeof(*rep_args));
2774 rep_args->is_delta = FALSE;
2776 if (strcmp(buffer, REP_PLAIN) == 0)
2778 *rep_args_p = rep_args;
2779 return SVN_NO_ERROR;
2782 if (strcmp(buffer, REP_DELTA) == 0)
2784 /* This is a delta against the empty stream. */
2785 rep_args->is_delta = TRUE;
2786 rep_args->is_delta_vs_empty = TRUE;
2787 *rep_args_p = rep_args;
2788 return SVN_NO_ERROR;
2791 rep_args->is_delta = TRUE;
2792 rep_args->is_delta_vs_empty = FALSE;
2794 /* We have hopefully a DELTA vs. a non-empty base revision. */
2795 str = svn_cstring_tokenize(" ", &last_str);
2796 if (! str || (strcmp(str, REP_DELTA) != 0))
2799 str = svn_cstring_tokenize(" ", &last_str);
2802 rep_args->base_revision = SVN_STR_TO_REV(str);
2804 str = svn_cstring_tokenize(" ", &last_str);
2807 SVN_ERR(svn_cstring_atoi64(&val, str));
2808 rep_args->base_offset = (apr_off_t)val;
2810 str = svn_cstring_tokenize(" ", &last_str);
2813 SVN_ERR(svn_cstring_atoi64(&val, str));
2814 rep_args->base_length = (svn_filesize_t)val;
2816 *rep_args_p = rep_args;
2817 return SVN_NO_ERROR;
2820 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2821 _("Malformed representation header at %s"),
2822 path_and_offset_of(file, pool));
2825 /* Given a revision file REV_FILE, opened to REV in FS, find the Node-ID
2826 of the header located at OFFSET and store it in *ID_P. Allocate
2827 temporary variables from POOL. */
2828 static svn_error_t *
2829 get_fs_id_at_offset(svn_fs_id_t **id_p,
2830 apr_file_t *rev_file,
2837 apr_hash_t *headers;
2838 const char *node_id_str;
2840 SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool));
2842 SVN_ERR(read_header_block(&headers,
2843 svn_stream_from_aprfile2(rev_file, TRUE, pool),
2846 /* In error messages, the offset is relative to the pack file,
2847 not to the rev file. */
2849 node_id_str = svn_hash_gets(headers, HEADER_ID);
2851 if (node_id_str == NULL)
2852 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2853 _("Missing node-id in node-rev at r%ld "
2856 apr_psprintf(pool, "%" APR_OFF_T_FMT, offset));
2858 id = svn_fs_fs__id_parse(node_id_str, strlen(node_id_str), pool);
2861 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2862 _("Corrupt node-id '%s' in node-rev at r%ld "
2865 apr_psprintf(pool, "%" APR_OFF_T_FMT, offset));
2869 /* ### assert that the txn_id is REV/OFFSET ? */
2871 return SVN_NO_ERROR;
2875 /* Given an open revision file REV_FILE in FS for REV, locate the trailer that
2876 specifies the offset to the root node-id and to the changed path
2877 information. Store the root node offset in *ROOT_OFFSET and the
2878 changed path offset in *CHANGES_OFFSET. If either of these
2879 pointers is NULL, do nothing with it.
2881 If PACKED is true, REV_FILE should be a packed shard file.
2882 ### There is currently no such parameter. This function assumes that
2883 is_packed_rev(FS, REV) will indicate whether REV_FILE is a packed
2884 file. Therefore FS->fsap_data->min_unpacked_rev must not have been
2885 refreshed since REV_FILE was opened if there is a possibility that
2886 revision REV may have become packed since then.
2887 TODO: Take an IS_PACKED parameter instead, in order to remove this
2890 Allocate temporary variables from POOL. */
2891 static svn_error_t *
2892 get_root_changes_offset(apr_off_t *root_offset,
2893 apr_off_t *changes_offset,
2894 apr_file_t *rev_file,
2899 fs_fs_data_t *ffd = fs->fsap_data;
2901 apr_off_t rev_offset;
2906 apr_seek_where_t seek_relative;
2908 /* Determine where to seek to in the file.
2910 If we've got a pack file, we want to seek to the end of the desired
2911 revision. But we don't track that, so we seek to the beginning of the
2914 Unless the next revision is in a different file, in which case, we can
2915 just seek to the end of the pack file -- just like we do in the
2917 if (is_packed_rev(fs, rev) && ((rev + 1) % ffd->max_files_per_dir != 0))
2919 SVN_ERR(get_packed_offset(&offset, fs, rev + 1, pool));
2920 seek_relative = APR_SET;
2924 seek_relative = APR_END;
2928 /* Offset of the revision from the start of the pack file, if applicable. */
2929 if (is_packed_rev(fs, rev))
2930 SVN_ERR(get_packed_offset(&rev_offset, fs, rev, pool));
2934 /* We will assume that the last line containing the two offsets
2935 will never be longer than 64 characters. */
2936 SVN_ERR(svn_io_file_seek(rev_file, seek_relative, &offset, pool));
2938 offset -= sizeof(buf);
2939 SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool));
2941 /* Read in this last block, from which we will identify the last line. */
2943 SVN_ERR(svn_io_file_read(rev_file, buf, &len, pool));
2945 /* This cast should be safe since the maximum amount read, 64, will
2946 never be bigger than the size of an int. */
2947 num_bytes = (int) len;
2949 /* The last byte should be a newline. */
2950 if (buf[num_bytes - 1] != '\n')
2952 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2953 _("Revision file (r%ld) lacks trailing newline"),
2957 /* Look for the next previous newline. */
2958 for (i = num_bytes - 2; i >= 0; i--)
2966 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2967 _("Final line in revision file (r%ld) longer "
2968 "than 64 characters"),
2975 /* find the next space */
2976 for ( ; i < (num_bytes - 2) ; i++)
2980 if (i == (num_bytes - 2))
2981 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2982 _("Final line in revision file r%ld missing space"),
2990 SVN_ERR(svn_cstring_atoi64(&val, str));
2991 *root_offset = rev_offset + (apr_off_t)val;
2997 /* find the next newline */
2998 for ( ; i < num_bytes; i++)
3007 SVN_ERR(svn_cstring_atoi64(&val, str));
3008 *changes_offset = rev_offset + (apr_off_t)val;
3011 return SVN_NO_ERROR;
3014 /* Move a file into place from OLD_FILENAME in the transactions
3015 directory to its final location NEW_FILENAME in the repository. On
3016 Unix, match the permissions of the new file to the permissions of
3017 PERMS_REFERENCE. Temporary allocations are from POOL.
3019 This function almost duplicates svn_io_file_move(), but it tries to
3020 guarantee a flush. */
3021 static svn_error_t *
3022 move_into_place(const char *old_filename,
3023 const char *new_filename,
3024 const char *perms_reference,
3029 SVN_ERR(svn_io_copy_perms(perms_reference, old_filename, pool));
3031 /* Move the file into place. */
3032 err = svn_io_file_rename(old_filename, new_filename, pool);
3033 if (err && APR_STATUS_IS_EXDEV(err->apr_err))
3037 /* Can't rename across devices; fall back to copying. */
3038 svn_error_clear(err);
3040 SVN_ERR(svn_io_copy_file(old_filename, new_filename, TRUE, pool));
3042 /* Flush the target of the copy to disk. */
3043 SVN_ERR(svn_io_file_open(&file, new_filename, APR_READ,
3044 APR_OS_DEFAULT, pool));
3045 /* ### BH: Does this really guarantee a flush of the data written
3046 ### via a completely different handle on all operating systems?
3048 ### Maybe we should perform the copy ourselves instead of making
3049 ### apr do that and flush the real handle? */
3050 SVN_ERR(svn_io_file_flush_to_disk(file, pool));
3051 SVN_ERR(svn_io_file_close(file, pool));
3054 return svn_error_trace(err);
3058 /* Linux has the unusual feature that fsync() on a file is not
3059 enough to ensure that a file's directory entries have been
3060 flushed to disk; you have to fsync the directory as well.
3061 On other operating systems, we'd only be asking for trouble
3062 by trying to open and fsync a directory. */
3063 const char *dirname;
3066 dirname = svn_dirent_dirname(new_filename, pool);
3067 SVN_ERR(svn_io_file_open(&file, dirname, APR_READ, APR_OS_DEFAULT,
3069 SVN_ERR(svn_io_file_flush_to_disk(file, pool));
3070 SVN_ERR(svn_io_file_close(file, pool));
3074 return SVN_NO_ERROR;
3078 svn_fs_fs__rev_get_root(svn_fs_id_t **root_id_p,
3083 fs_fs_data_t *ffd = fs->fsap_data;
3084 apr_file_t *revision_file;
3085 apr_off_t root_offset;
3086 svn_fs_id_t *root_id = NULL;
3087 svn_boolean_t is_cached;
3089 SVN_ERR(ensure_revision_exists(fs, rev, pool));
3091 SVN_ERR(svn_cache__get((void **) root_id_p, &is_cached,
3092 ffd->rev_root_id_cache, &rev, pool));
3094 return SVN_NO_ERROR;
3096 SVN_ERR(open_pack_or_rev_file(&revision_file, fs, rev, pool));
3097 SVN_ERR(get_root_changes_offset(&root_offset, NULL, revision_file, fs, rev,
3100 SVN_ERR(get_fs_id_at_offset(&root_id, revision_file, fs, rev,
3101 root_offset, pool));
3103 SVN_ERR(svn_io_file_close(revision_file, pool));
3105 SVN_ERR(svn_cache__set(ffd->rev_root_id_cache, &rev, root_id, pool));
3107 *root_id_p = root_id;
3109 return SVN_NO_ERROR;
3112 /* Revprop caching management.
3117 * Revprop caching needs to be activated and will be deactivated for the
3118 * respective FS instance if the necessary infrastructure could not be
3119 * initialized. In deactivated mode, there is almost no runtime overhead
3120 * associated with revprop caching. As long as no revprops are being read
3121 * or changed, revprop caching imposes no overhead.
3123 * When activated, we cache revprops using (revision, generation) pairs
3124 * as keys with the generation being incremented upon every revprop change.
3125 * Since the cache is process-local, the generation needs to be tracked
3126 * for at least as long as the process lives but may be reset afterwards.
3128 * To track the revprop generation, we use two-layer approach. On the lower
3129 * level, we use named atomics to have a system-wide consistent value for
3130 * the current revprop generation. However, those named atomics will only
3131 * remain valid for as long as at least one process / thread in the system
3132 * accesses revprops in the respective repository. The underlying shared
3133 * memory gets cleaned up afterwards.
3135 * On the second level, we will use a persistent file to track the latest
3136 * revprop generation. It will be written upon each revprop change but
3137 * only be read if we are the first process to initialize the named atomics
3140 * The overhead for the second and following accesses to revprops is
3141 * almost zero on most systems.
3147 * A problem is that we need to provide a globally available file name to
3148 * back the SHM implementation on OSes that need it. We can only assume
3149 * write access to some file within the respective repositories. Because
3150 * a given server process may access thousands of repositories during its
3151 * lifetime, keeping the SHM data alive for all of them is also not an
3154 * So, we store the new revprop generation on disk as part of each
3155 * setrevprop call, i.e. this write will be serialized and the write order
3156 * be guaranteed by the repository write lock.
3158 * The only racy situation occurs when the data is being read again by two
3159 * processes concurrently but in that situation, the first process to
3160 * finish that procedure is guaranteed to be the only one that initializes
3161 * the SHM data. Since even writers will first go through that
3162 * initialization phase, they will never operate on stale data.
3165 /* Read revprop generation as stored on disk for repository FS. The result
3166 * is returned in *CURRENT. Default to 2 if no such file is available.
3168 static svn_error_t *
3169 read_revprop_generation_file(apr_int64_t *current,
3177 const char *path = path_revprop_generation(fs, pool);
3179 err = svn_io_file_open(&file, path,
3180 APR_READ | APR_BUFFERED,
3181 APR_OS_DEFAULT, pool);
3182 if (err && APR_STATUS_IS_ENOENT(err->apr_err))
3184 svn_error_clear(err);
3187 return SVN_NO_ERROR;
3192 SVN_ERR(svn_io_read_length_line(file, buf, &len, pool));
3194 /* Check that the first line contains only digits. */
3195 SVN_ERR(check_file_buffer_numeric(buf, 0, path,
3196 "Revprop Generation", pool));
3197 SVN_ERR(svn_cstring_atoi64(current, buf));
3199 return svn_io_file_close(file, pool);
3202 /* Write the CURRENT revprop generation to disk for repository FS.
3204 static svn_error_t *
3205 write_revprop_generation_file(svn_fs_t *fs,
3206 apr_int64_t current,
3210 const char *tmp_path;
3212 char buf[SVN_INT64_BUFFER_SIZE];
3213 apr_size_t len = svn__i64toa(buf, current);
3216 SVN_ERR(svn_io_open_unique_file3(&file, &tmp_path, fs->path,
3217 svn_io_file_del_none, pool, pool));
3218 SVN_ERR(svn_io_file_write_full(file, buf, len + 1, NULL, pool));
3219 SVN_ERR(svn_io_file_close(file, pool));
3221 return move_into_place(tmp_path, path_revprop_generation(fs, pool),
3225 /* Make sure the revprop_namespace member in FS is set. */
3226 static svn_error_t *
3227 ensure_revprop_namespace(svn_fs_t *fs)
3229 fs_fs_data_t *ffd = fs->fsap_data;
3231 return ffd->revprop_namespace == NULL
3232 ? svn_atomic_namespace__create(&ffd->revprop_namespace,
3233 svn_dirent_join(fs->path,
3234 ATOMIC_REVPROP_NAMESPACE,
3240 /* Make sure the revprop_namespace member in FS is set. */
3241 static svn_error_t *
3242 cleanup_revprop_namespace(svn_fs_t *fs)
3244 const char *name = svn_dirent_join(fs->path,
3245 ATOMIC_REVPROP_NAMESPACE,
3247 return svn_error_trace(svn_atomic_namespace__cleanup(name, fs->pool));
3250 /* Make sure the revprop_generation member in FS is set and, if necessary,
3251 * initialized with the latest value stored on disk.
3253 static svn_error_t *
3254 ensure_revprop_generation(svn_fs_t *fs, apr_pool_t *pool)
3256 fs_fs_data_t *ffd = fs->fsap_data;
3258 SVN_ERR(ensure_revprop_namespace(fs));
3259 if (ffd->revprop_generation == NULL)
3261 apr_int64_t current = 0;
3263 SVN_ERR(svn_named_atomic__get(&ffd->revprop_generation,
3264 ffd->revprop_namespace,
3265 ATOMIC_REVPROP_GENERATION,
3268 /* If the generation is at 0, we just created a new namespace
3269 * (it would be at least 2 otherwise). Read the latest generation
3270 * from disk and if we are the first one to initialize the atomic
3271 * (i.e. is still 0), set it to the value just gotten.
3273 SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation));
3276 SVN_ERR(read_revprop_generation_file(¤t, fs, pool));
3277 SVN_ERR(svn_named_atomic__cmpxchg(NULL, current, 0,
3278 ffd->revprop_generation));
3282 return SVN_NO_ERROR;
3285 /* Make sure the revprop_timeout member in FS is set. */
3286 static svn_error_t *
3287 ensure_revprop_timeout(svn_fs_t *fs)
3289 fs_fs_data_t *ffd = fs->fsap_data;
3291 SVN_ERR(ensure_revprop_namespace(fs));
3292 return ffd->revprop_timeout == NULL
3293 ? svn_named_atomic__get(&ffd->revprop_timeout,
3294 ffd->revprop_namespace,
3295 ATOMIC_REVPROP_TIMEOUT,
3300 /* Create an error object with the given MESSAGE and pass it to the
3301 WARNING member of FS. */
3303 log_revprop_cache_init_warning(svn_fs_t *fs,
3304 svn_error_t *underlying_err,
3305 const char *message)
3307 svn_error_t *err = svn_error_createf(SVN_ERR_FS_REVPROP_CACHE_INIT_FAILURE,
3312 (fs->warning)(fs->warning_baton, err);
3314 svn_error_clear(err);
3317 /* Test whether revprop cache and necessary infrastructure are
3319 static svn_boolean_t
3320 has_revprop_cache(svn_fs_t *fs, apr_pool_t *pool)
3322 fs_fs_data_t *ffd = fs->fsap_data;
3325 /* is the cache (still) enabled? */
3326 if (ffd->revprop_cache == NULL)
3329 /* is it efficient? */
3330 if (!svn_named_atomic__is_efficient())
3332 /* access to it would be quite slow
3333 * -> disable the revprop cache for good
3335 ffd->revprop_cache = NULL;
3336 log_revprop_cache_init_warning(fs, NULL,
3337 "Revprop caching for '%s' disabled"
3338 " because it would be inefficient.");
3343 /* try to access our SHM-backed infrastructure */
3344 error = ensure_revprop_generation(fs, pool);
3347 /* failure -> disable revprop cache for good */
3349 ffd->revprop_cache = NULL;
3350 log_revprop_cache_init_warning(fs, error,
3351 "Revprop caching for '%s' disabled "
3352 "because SHM infrastructure for revprop "
3353 "caching failed to initialize.");
3361 /* Baton structure for revprop_generation_fixup. */
3362 typedef struct revprop_generation_fixup_t
3364 /* revprop generation to read */
3365 apr_int64_t *generation;
3367 /* containing the revprop_generation member to query */
3369 } revprop_generation_upgrade_t;
3371 /* If the revprop generation has an odd value, it means the original writer
3372 of the revprop got killed. We don't know whether that process as able
3373 to change the revprop data but we assume that it was. Therefore, we
3374 increase the generation in that case to basically invalidate everyones
3376 Execute this onlx while holding the write lock to the repo in baton->FFD.
3378 static svn_error_t *
3379 revprop_generation_fixup(void *void_baton,
3382 revprop_generation_upgrade_t *baton = void_baton;
3383 assert(baton->ffd->has_write_lock);
3385 /* Maybe, either the original revprop writer or some other reader has
3386 already corrected / bumped the revprop generation. Thus, we need
3387 to read it again. */
3388 SVN_ERR(svn_named_atomic__read(baton->generation,
3389 baton->ffd->revprop_generation));
3391 /* Cause everyone to re-read revprops upon their next access, if the
3392 last revprop write did not complete properly. */
3393 while (*baton->generation % 2)
3394 SVN_ERR(svn_named_atomic__add(baton->generation,
3396 baton->ffd->revprop_generation));
3398 return SVN_NO_ERROR;
3401 /* Read the current revprop generation and return it in *GENERATION.
3402 Also, detect aborted / crashed writers and recover from that.
3403 Use the access object in FS to set the shared mem values. */
3404 static svn_error_t *
3405 read_revprop_generation(apr_int64_t *generation,
3409 apr_int64_t current = 0;
3410 fs_fs_data_t *ffd = fs->fsap_data;
3412 /* read the current revprop generation number */
3413 SVN_ERR(ensure_revprop_generation(fs, pool));
3414 SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation));
3416 /* is an unfinished revprop write under the way? */
3419 apr_int64_t timeout = 0;
3421 /* read timeout for the write operation */
3422 SVN_ERR(ensure_revprop_timeout(fs));
3423 SVN_ERR(svn_named_atomic__read(&timeout, ffd->revprop_timeout));
3425 /* has the writer process been aborted,
3426 * i.e. has the timeout been reached?
3428 if (apr_time_now() > timeout)
3430 revprop_generation_upgrade_t baton;
3431 baton.generation = ¤t;
3434 /* Ensure that the original writer process no longer exists by
3435 * acquiring the write lock to this repository. Then, fix up
3436 * the revprop generation.
3438 if (ffd->has_write_lock)
3439 SVN_ERR(revprop_generation_fixup(&baton, pool));
3441 SVN_ERR(svn_fs_fs__with_write_lock(fs, revprop_generation_fixup,
3446 /* return the value we just got */
3447 *generation = current;
3448 return SVN_NO_ERROR;
3451 /* Set the revprop generation to the next odd number to indicate that
3452 there is a revprop write process under way. If that times out,
3453 readers shall recover from that state & re-read revprops.
3454 Use the access object in FS to set the shared mem value. */
3455 static svn_error_t *
3456 begin_revprop_change(svn_fs_t *fs, apr_pool_t *pool)
3458 apr_int64_t current;
3459 fs_fs_data_t *ffd = fs->fsap_data;
3461 /* set the timeout for the write operation */
3462 SVN_ERR(ensure_revprop_timeout(fs));
3463 SVN_ERR(svn_named_atomic__write(NULL,
3464 apr_time_now() + REVPROP_CHANGE_TIMEOUT,
3465 ffd->revprop_timeout));
3467 /* set the revprop generation to an odd value to indicate
3468 * that a write is in progress
3470 SVN_ERR(ensure_revprop_generation(fs, pool));
3473 SVN_ERR(svn_named_atomic__add(¤t,
3475 ffd->revprop_generation));
3477 while (current % 2 == 0);
3479 return SVN_NO_ERROR;
3482 /* Set the revprop generation to the next even number to indicate that
3483 a) readers shall re-read revprops, and
3484 b) the write process has been completed (no recovery required)
3485 Use the access object in FS to set the shared mem value. */
3486 static svn_error_t *
3487 end_revprop_change(svn_fs_t *fs, apr_pool_t *pool)
3489 apr_int64_t current = 1;
3490 fs_fs_data_t *ffd = fs->fsap_data;
3492 /* set the revprop generation to an even value to indicate
3493 * that a write has been completed
3495 SVN_ERR(ensure_revprop_generation(fs, pool));
3498 SVN_ERR(svn_named_atomic__add(¤t,
3500 ffd->revprop_generation));
3502 while (current % 2);
3504 /* Save the latest generation to disk. FS is currently in a "locked"
3505 * state such that we can be sure the be the only ones to write that
3508 return write_revprop_generation_file(fs, current, pool);
3511 /* Container for all data required to access the packed revprop file
3512 * for a given REVISION. This structure will be filled incrementally
3513 * by read_pack_revprops() its sub-routines.
3515 typedef struct packed_revprops_t
3517 /* revision number to read (not necessarily the first in the pack) */
3518 svn_revnum_t revision;
3520 /* current revprop generation. Used when populating the revprop cache */
3521 apr_int64_t generation;
3523 /* the actual revision properties */
3524 apr_hash_t *properties;
3526 /* their size when serialized to a single string
3527 * (as found in PACKED_REVPROPS) */
3528 apr_size_t serialized_size;
3531 /* name of the pack file (without folder path) */
3532 const char *filename;
3534 /* packed shard folder path */
3537 /* sum of values in SIZES */
3538 apr_size_t total_size;
3540 /* first revision in the pack (>= MANIFEST_START) */
3541 svn_revnum_t start_revision;
3543 /* size of the revprops in PACKED_REVPROPS */
3544 apr_array_header_t *sizes;
3546 /* offset of the revprops in PACKED_REVPROPS */
3547 apr_array_header_t *offsets;
3550 /* concatenation of the serialized representation of all revprops
3551 * in the pack, i.e. the pack content without header and compression */
3552 svn_stringbuf_t *packed_revprops;
3554 /* First revision covered by MANIFEST.
3555 * Will equal the shard start revision or 1, for the 1st shard. */
3556 svn_revnum_t manifest_start;
3558 /* content of the manifest.
3559 * Maps long(rev - MANIFEST_START) to const char* pack file name */
3560 apr_array_header_t *manifest;
3561 } packed_revprops_t;
3563 /* Parse the serialized revprops in CONTENT and return them in *PROPERTIES.
3564 * Also, put them into the revprop cache, if activated, for future use.
3565 * Three more parameters are being used to update the revprop cache: FS is
3566 * our file system, the revprops belong to REVISION and the global revprop
3567 * GENERATION is used as well.
3569 * The returned hash will be allocated in POOL, SCRATCH_POOL is being used
3570 * for temporary allocations.
3572 static svn_error_t *
3573 parse_revprop(apr_hash_t **properties,
3575 svn_revnum_t revision,
3576 apr_int64_t generation,
3577 svn_string_t *content,
3579 apr_pool_t *scratch_pool)
3581 svn_stream_t *stream = svn_stream_from_string(content, scratch_pool);
3582 *properties = apr_hash_make(pool);
3584 SVN_ERR(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool));
3585 if (has_revprop_cache(fs, pool))
3587 fs_fs_data_t *ffd = fs->fsap_data;
3588 pair_cache_key_t key = { 0 };
3590 key.revision = revision;
3591 key.second = generation;
3592 SVN_ERR(svn_cache__set(ffd->revprop_cache, &key, *properties,
3596 return SVN_NO_ERROR;
3599 /* Read the non-packed revprops for revision REV in FS, put them into the
3600 * revprop cache if activated and return them in *PROPERTIES. GENERATION
3601 * is the current revprop generation.
3603 * If the data could not be read due to an otherwise recoverable error,
3604 * leave *PROPERTIES unchanged. No error will be returned in that case.
3606 * Allocations will be done in POOL.
3608 static svn_error_t *
3609 read_non_packed_revprop(apr_hash_t **properties,
3612 apr_int64_t generation,
3615 svn_stringbuf_t *content = NULL;
3616 apr_pool_t *iterpool = svn_pool_create(pool);
3617 svn_boolean_t missing = FALSE;
3620 for (i = 0; i < RECOVERABLE_RETRY_COUNT && !missing && !content; ++i)
3622 svn_pool_clear(iterpool);
3623 SVN_ERR(try_stringbuf_from_file(&content,
3625 path_revprops(fs, rev, iterpool),
3626 i + 1 < RECOVERABLE_RETRY_COUNT,
3631 SVN_ERR(parse_revprop(properties, fs, rev, generation,
3632 svn_stringbuf__morph_into_string(content),
3635 svn_pool_clear(iterpool);
3637 return SVN_NO_ERROR;
3640 /* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST
3641 * members. Use POOL for allocating results and SCRATCH_POOL for temporaries.
3643 static svn_error_t *
3644 get_revprop_packname(svn_fs_t *fs,
3645 packed_revprops_t *revprops,
3647 apr_pool_t *scratch_pool)
3649 fs_fs_data_t *ffd = fs->fsap_data;
3650 svn_stringbuf_t *content = NULL;
3651 const char *manifest_file_path;
3654 /* read content of the manifest file */
3655 revprops->folder = path_revprops_pack_shard(fs, revprops->revision, pool);
3656 manifest_file_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool);
3658 SVN_ERR(read_content(&content, manifest_file_path, pool));
3660 /* parse the manifest. Every line is a file name */
3661 revprops->manifest = apr_array_make(pool, ffd->max_files_per_dir,
3662 sizeof(const char*));
3664 /* Read all lines. Since the last line ends with a newline, we will
3665 end up with a valid but empty string after the last entry. */
3666 while (content->data && *content->data)
3668 APR_ARRAY_PUSH(revprops->manifest, const char*) = content->data;
3669 content->data = strchr(content->data, '\n');
3677 /* Index for our revision. Rev 0 is excluded from the first shard. */
3678 revprops->manifest_start = revprops->revision
3679 - (revprops->revision % ffd->max_files_per_dir);
3680 if (revprops->manifest_start == 0)
3681 ++revprops->manifest_start;
3682 idx = (int)(revprops->revision - revprops->manifest_start);
3684 if (revprops->manifest->nelts <= idx)
3685 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
3686 _("Packed revprop manifest for r%ld too "
3687 "small"), revprops->revision);
3689 /* Now get the file name */
3690 revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*);
3692 return SVN_NO_ERROR;
3695 /* Return TRUE, if revision R1 and R2 refer to the same shard in FS.
3697 static svn_boolean_t
3698 same_shard(svn_fs_t *fs,
3702 fs_fs_data_t *ffd = fs->fsap_data;
3703 return (r1 / ffd->max_files_per_dir) == (r2 / ffd->max_files_per_dir);
3706 /* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS,
3707 * fill the START_REVISION, SIZES, OFFSETS members. Also, make
3708 * PACKED_REVPROPS point to the first serialized revprop.
3710 * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as
3711 * well as the SERIALIZED_SIZE member. If revprop caching has been
3712 * enabled, parse all revprops in the pack and cache them.
3714 static svn_error_t *
3715 parse_packed_revprops(svn_fs_t *fs,
3716 packed_revprops_t *revprops,
3718 apr_pool_t *scratch_pool)
3720 svn_stream_t *stream;
3721 apr_int64_t first_rev, count, i;
3723 const char *header_end;
3724 apr_pool_t *iterpool = svn_pool_create(scratch_pool);
3726 /* decompress (even if the data is only "stored", there is still a
3727 * length header to remove) */
3728 svn_string_t *compressed
3729 = svn_stringbuf__morph_into_string(revprops->packed_revprops);
3730 svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(pool);
3731 SVN_ERR(svn__decompress(compressed, uncompressed, APR_SIZE_MAX));
3733 /* read first revision number and number of revisions in the pack */
3734 stream = svn_stream_from_stringbuf(uncompressed, scratch_pool);
3735 SVN_ERR(read_number_from_stream(&first_rev, NULL, stream, iterpool));
3736 SVN_ERR(read_number_from_stream(&count, NULL, stream, iterpool));
3738 /* Check revision range for validity. */
3739 if ( !same_shard(fs, revprops->revision, first_rev)
3740 || !same_shard(fs, revprops->revision, first_rev + count - 1)
3742 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
3743 _("Revprop pack for revision r%ld"
3744 " contains revprops for r%ld .. r%ld"),
3746 (svn_revnum_t)first_rev,
3747 (svn_revnum_t)(first_rev + count -1));
3749 /* Since start & end are in the same shard, it is enough to just test
3750 * the FIRST_REV for being actually packed. That will also cover the
3751 * special case of rev 0 never being packed. */
3752 if (!is_packed_revprop(fs, first_rev))
3753 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
3754 _("Revprop pack for revision r%ld"
3755 " starts at non-packed revisions r%ld"),
3756 revprops->revision, (svn_revnum_t)first_rev);
3758 /* make PACKED_REVPROPS point to the first char after the header.
3759 * This is where the serialized revprops are. */
3760 header_end = strstr(uncompressed->data, "\n\n");
3761 if (header_end == NULL)
3762 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
3763 _("Header end not found"));
3765 offset = header_end - uncompressed->data + 2;
3767 revprops->packed_revprops = svn_stringbuf_create_empty(pool);
3768 revprops->packed_revprops->data = uncompressed->data + offset;
3769 revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset);
3770 revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset);
3772 /* STREAM still points to the first entry in the sizes list.
3773 * Init / construct REVPROPS members. */
3774 revprops->start_revision = (svn_revnum_t)first_rev;
3775 revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset));
3776 revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset));
3778 /* Now parse, revision by revision, the size and content of each
3779 * revisions' revprops. */
3780 for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i)
3783 svn_string_t serialized;
3784 apr_hash_t *properties;
3785 svn_revnum_t revision = (svn_revnum_t)(first_rev + i);
3787 /* read & check the serialized size */
3788 SVN_ERR(read_number_from_stream(&size, NULL, stream, iterpool));
3789 if (size + offset > (apr_int64_t)revprops->packed_revprops->len)
3790 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
3791 _("Packed revprop size exceeds pack file size"));
3793 /* Parse this revprops list, if necessary */
3794 serialized.data = revprops->packed_revprops->data + offset;
3795 serialized.len = (apr_size_t)size;
3797 if (revision == revprops->revision)
3799 SVN_ERR(parse_revprop(&revprops->properties, fs, revision,
3800 revprops->generation, &serialized,
3802 revprops->serialized_size = serialized.len;
3806 /* If revprop caching is enabled, parse any revprops.
3807 * They will get cached as a side-effect of this. */
3808 if (has_revprop_cache(fs, pool))
3809 SVN_ERR(parse_revprop(&properties, fs, revision,
3810 revprops->generation, &serialized,
3811 iterpool, iterpool));
3814 /* fill REVPROPS data structures */
3815 APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len;
3816 APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset;
3817 revprops->total_size += serialized.len;
3819 offset += serialized.len;
3821 svn_pool_clear(iterpool);
3824 return SVN_NO_ERROR;
3827 /* In filesystem FS, read the packed revprops for revision REV into
3828 * *REVPROPS. Use GENERATION to populate the revprop cache, if enabled.
3829 * Allocate data in POOL.
3831 static svn_error_t *
3832 read_pack_revprop(packed_revprops_t **revprops,
3835 apr_int64_t generation,
3838 apr_pool_t *iterpool = svn_pool_create(pool);
3839 svn_boolean_t missing = FALSE;
3841 packed_revprops_t *result;
3844 /* someone insisted that REV is packed. Double-check if necessary */
3845 if (!is_packed_revprop(fs, rev))
3846 SVN_ERR(update_min_unpacked_rev(fs, iterpool));
3848 if (!is_packed_revprop(fs, rev))
3849 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
3850 _("No such packed revision %ld"), rev);
3852 /* initialize the result data structure */
3853 result = apr_pcalloc(pool, sizeof(*result));
3854 result->revision = rev;
3855 result->generation = generation;
3857 /* try to read the packed revprops. This may require retries if we have
3858 * concurrent writers. */
3859 for (i = 0; i < RECOVERABLE_RETRY_COUNT && !result->packed_revprops; ++i)
3861 const char *file_path;
3863 /* there might have been concurrent writes.
3864 * Re-read the manifest and the pack file.
3866 SVN_ERR(get_revprop_packname(fs, result, pool, iterpool));
3867 file_path = svn_dirent_join(result->folder,
3870 SVN_ERR(try_stringbuf_from_file(&result->packed_revprops,
3873 i + 1 < RECOVERABLE_RETRY_COUNT,
3876 /* If we could not find the file, there was a write.
3877 * So, we should refresh our revprop generation info as well such
3878 * that others may find data we will put into the cache. They would
3879 * consider it outdated, otherwise.
3881 if (missing && has_revprop_cache(fs, pool))
3882 SVN_ERR(read_revprop_generation(&result->generation, fs, pool));
3884 svn_pool_clear(iterpool);
3887 /* the file content should be available now */
3888 if (!result->packed_revprops)
3889 return svn_error_createf(SVN_ERR_FS_PACKED_REVPROP_READ_FAILURE, NULL,
3890 _("Failed to read revprop pack file for r%ld"), rev);
3892 /* parse it. RESULT will be complete afterwards. */
3893 err = parse_packed_revprops(fs, result, pool, iterpool);
3894 svn_pool_destroy(iterpool);
3896 return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
3897 _("Revprop pack file for r%ld is corrupt"), rev);
3901 return SVN_NO_ERROR;
3904 /* Read the revprops for revision REV in FS and return them in *PROPERTIES_P.
3906 * Allocations will be done in POOL.
3908 static svn_error_t *
3909 get_revision_proplist(apr_hash_t **proplist_p,
3914 fs_fs_data_t *ffd = fs->fsap_data;
3915 apr_int64_t generation = 0;
3917 /* not found, yet */
3920 /* should they be available at all? */
3921 SVN_ERR(ensure_revision_exists(fs, rev, pool));
3923 /* Try cache lookup first. */
3924 if (has_revprop_cache(fs, pool))
3926 svn_boolean_t is_cached;
3927 pair_cache_key_t key = { 0 };
3929 SVN_ERR(read_revprop_generation(&generation, fs, pool));
3932 key.second = generation;
3933 SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached,
3934 ffd->revprop_cache, &key, pool));
3936 return SVN_NO_ERROR;
3939 /* if REV had not been packed when we began, try reading it from the
3940 * non-packed shard. If that fails, we will fall through to packed
3942 if (!is_packed_revprop(fs, rev))
3944 svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev,
3948 if (!APR_STATUS_IS_ENOENT(err->apr_err)
3949 || ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
3950 return svn_error_trace(err);
3952 svn_error_clear(err);
3953 *proplist_p = NULL; /* in case read_non_packed_revprop changed it */
3957 /* if revprop packing is available and we have not read the revprops, yet,
3958 * try reading them from a packed shard. If that fails, REV is most
3959 * likely invalid (or its revprops highly contested). */
3960 if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p)
3962 packed_revprops_t *packed_revprops;
3963 SVN_ERR(read_pack_revprop(&packed_revprops, fs, rev, generation, pool));
3964 *proplist_p = packed_revprops->properties;
3967 /* The revprops should have been there. Did we get them? */
3969 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
3970 _("Could not read revprops for revision %ld"),
3973 return SVN_NO_ERROR;
3976 /* Serialize the revision property list PROPLIST of revision REV in
3977 * filesystem FS to a non-packed file. Return the name of that temporary
3978 * file in *TMP_PATH and the file path that it must be moved to in
3981 * Use POOL for allocations.
3983 static svn_error_t *
3984 write_non_packed_revprop(const char **final_path,
3985 const char **tmp_path,
3988 apr_hash_t *proplist,
3991 svn_stream_t *stream;
3992 *final_path = path_revprops(fs, rev, pool);
3994 /* ### do we have a directory sitting around already? we really shouldn't
3995 ### have to get the dirname here. */
3996 SVN_ERR(svn_stream_open_unique(&stream, tmp_path,
3997 svn_dirent_dirname(*final_path, pool),
3998 svn_io_file_del_none, pool, pool));
3999 SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
4000 SVN_ERR(svn_stream_close(stream));
4002 return SVN_NO_ERROR;
4005 /* After writing the new revprop file(s), call this function to move the
4006 * file at TMP_PATH to FINAL_PATH and give it the permissions from
4009 * If indicated in BUMP_GENERATION, increase FS' revprop generation.
4010 * Finally, delete all the temporary files given in FILES_TO_DELETE.
4011 * The latter may be NULL.
4013 * Use POOL for temporary allocations.
4015 static svn_error_t *
4016 switch_to_new_revprop(svn_fs_t *fs,
4017 const char *final_path,
4018 const char *tmp_path,
4019 const char *perms_reference,
4020 apr_array_header_t *files_to_delete,
4021 svn_boolean_t bump_generation,
4024 /* Now, we may actually be replacing revprops. Make sure that all other
4025 threads and processes will know about this. */
4026 if (bump_generation)
4027 SVN_ERR(begin_revprop_change(fs, pool));
4029 SVN_ERR(move_into_place(tmp_path, final_path, perms_reference, pool));
4031 /* Indicate that the update (if relevant) has been completed. */
4032 if (bump_generation)
4033 SVN_ERR(end_revprop_change(fs, pool));
4035 /* Clean up temporary files, if necessary. */
4036 if (files_to_delete)
4038 apr_pool_t *iterpool = svn_pool_create(pool);
4041 for (i = 0; i < files_to_delete->nelts; ++i)
4043 const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*);
4044 SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
4045 svn_pool_clear(iterpool);
4048 svn_pool_destroy(iterpool);
4050 return SVN_NO_ERROR;
4053 /* Write a pack file header to STREAM that starts at revision START_REVISION
4054 * and contains the indexes [START,END) of SIZES.
4056 static svn_error_t *
4057 serialize_revprops_header(svn_stream_t *stream,
4058 svn_revnum_t start_revision,
4059 apr_array_header_t *sizes,
4064 apr_pool_t *iterpool = svn_pool_create(pool);
4067 SVN_ERR_ASSERT(start < end);
4069 /* start revision and entry count */
4070 SVN_ERR(svn_stream_printf(stream, pool, "%ld\n", start_revision));
4071 SVN_ERR(svn_stream_printf(stream, pool, "%d\n", end - start));
4073 /* the sizes array */
4074 for (i = start; i < end; ++i)
4076 apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t);
4077 SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n",
4081 /* the double newline char indicates the end of the header */
4082 SVN_ERR(svn_stream_printf(stream, iterpool, "\n"));
4084 svn_pool_clear(iterpool);
4085 return SVN_NO_ERROR;
4088 /* Writes the a pack file to FILE_STREAM. It copies the serialized data
4089 * from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX.
4091 * The data for the latter is taken from NEW_SERIALIZED. Note, that
4092 * CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is
4093 * taken in that case but only a subset of the old data will be copied.
4095 * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size.
4096 * POOL is used for temporary allocations.
4098 static svn_error_t *
4099 repack_revprops(svn_fs_t *fs,
4100 packed_revprops_t *revprops,
4104 svn_stringbuf_t *new_serialized,
4105 apr_off_t new_total_size,
4106 svn_stream_t *file_stream,
4109 fs_fs_data_t *ffd = fs->fsap_data;
4110 svn_stream_t *stream;
4113 /* create data empty buffers and the stream object */
4114 svn_stringbuf_t *uncompressed
4115 = svn_stringbuf_create_ensure((apr_size_t)new_total_size, pool);
4116 svn_stringbuf_t *compressed
4117 = svn_stringbuf_create_empty(pool);
4118 stream = svn_stream_from_stringbuf(uncompressed, pool);
4120 /* write the header*/
4121 SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start,
4122 revprops->sizes, start, end, pool));
4124 /* append the serialized revprops */
4125 for (i = start; i < end; ++i)
4126 if (i == changed_index)
4128 SVN_ERR(svn_stream_write(stream,
4129 new_serialized->data,
4130 &new_serialized->len));
4135 = (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t);
4137 = (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t);
4139 SVN_ERR(svn_stream_write(stream,
4140 revprops->packed_revprops->data + offset,
4144 /* flush the stream buffer (if any) to our underlying data buffer */
4145 SVN_ERR(svn_stream_close(stream));
4147 /* compress / store the data */
4148 SVN_ERR(svn__compress(svn_stringbuf__morph_into_string(uncompressed),
4150 ffd->compress_packed_revprops
4151 ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
4152 : SVN_DELTA_COMPRESSION_LEVEL_NONE));
4154 /* finally, write the content to the target stream and close it */
4155 SVN_ERR(svn_stream_write(file_stream, compressed->data, &compressed->len));
4156 SVN_ERR(svn_stream_close(file_stream));
4158 return SVN_NO_ERROR;
4161 /* Allocate a new pack file name for revisions
4162 * [REVPROPS->START_REVISION + START, REVPROPS->START_REVISION + END - 1]
4163 * of REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE,
4164 * auto-create that array if necessary. Return an open file stream to
4165 * the new file in *STREAM allocated in POOL.
4167 static svn_error_t *
4168 repack_stream_open(svn_stream_t **stream,
4170 packed_revprops_t *revprops,
4173 apr_array_header_t **files_to_delete,
4177 const char *tag_string;
4178 svn_string_t *new_filename;
4182 = (int)(revprops->start_revision - revprops->manifest_start);
4184 /* get the old (= current) file name and enlist it for later deletion */
4185 const char *old_filename = APR_ARRAY_IDX(revprops->manifest,
4186 start + manifest_offset,
4189 if (*files_to_delete == NULL)
4190 *files_to_delete = apr_array_make(pool, 3, sizeof(const char*));
4192 APR_ARRAY_PUSH(*files_to_delete, const char*)
4193 = svn_dirent_join(revprops->folder, old_filename, pool);
4195 /* increase the tag part, i.e. the counter after the dot */
4196 tag_string = strchr(old_filename, '.');
4197 if (tag_string == NULL)
4198 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
4199 _("Packed file '%s' misses a tag"),
4202 SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1));
4203 new_filename = svn_string_createf(pool, "%ld.%" APR_INT64_T_FMT,
4204 revprops->start_revision + start,
4207 /* update the manifest to point to the new file */
4208 for (i = start; i < end; ++i)
4209 APR_ARRAY_IDX(revprops->manifest, i + manifest_offset, const char*)
4210 = new_filename->data;
4212 /* create a file stream for the new file */
4213 SVN_ERR(svn_io_file_open(&file, svn_dirent_join(revprops->folder,
4216 APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool));
4217 *stream = svn_stream_from_aprfile2(file, FALSE, pool);
4219 return SVN_NO_ERROR;
4222 /* For revision REV in filesystem FS, set the revision properties to
4223 * PROPLIST. Return a new file in *TMP_PATH that the caller shall move
4224 * to *FINAL_PATH to make the change visible. Files to be deleted will
4225 * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated.
4226 * Use POOL for allocations.
4228 static svn_error_t *
4229 write_packed_revprop(const char **final_path,
4230 const char **tmp_path,
4231 apr_array_header_t **files_to_delete,
4234 apr_hash_t *proplist,
4237 fs_fs_data_t *ffd = fs->fsap_data;
4238 packed_revprops_t *revprops;
4239 apr_int64_t generation = 0;
4240 svn_stream_t *stream;
4241 svn_stringbuf_t *serialized;
4242 apr_off_t new_total_size;
4245 /* read the current revprop generation. This value will not change
4246 * while we hold the global write lock to this FS. */
4247 if (has_revprop_cache(fs, pool))
4248 SVN_ERR(read_revprop_generation(&generation, fs, pool));
4250 /* read contents of the current pack file */
4251 SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, pool));
4253 /* serialize the new revprops */
4254 serialized = svn_stringbuf_create_empty(pool);
4255 stream = svn_stream_from_stringbuf(serialized, pool);
4256 SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
4257 SVN_ERR(svn_stream_close(stream));
4259 /* calculate the size of the new data */
4260 changed_index = (int)(rev - revprops->start_revision);
4261 new_total_size = revprops->total_size - revprops->serialized_size
4263 + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE;
4265 APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len;
4267 /* can we put the new data into the same pack as the before? */
4268 if ( new_total_size < ffd->revprop_pack_size
4269 || revprops->sizes->nelts == 1)
4271 /* simply replace the old pack file with new content as we do it
4272 * in the non-packed case */
4274 *final_path = svn_dirent_join(revprops->folder, revprops->filename,
4276 SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder,
4277 svn_io_file_del_none, pool, pool));
4278 SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts,
4279 changed_index, serialized, new_total_size,
4284 /* split the pack file into two of roughly equal size */
4285 int right_count, left_count, i;
4288 int right = revprops->sizes->nelts - 1;
4289 apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE;
4290 apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE;
4292 /* let left and right side grow such that their size difference
4293 * is minimal after each step. */
4294 while (left <= right)
4295 if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t)
4296 < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t))
4298 left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t)
4299 + SVN_INT64_BUFFER_SIZE;
4304 right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)
4305 + SVN_INT64_BUFFER_SIZE;
4309 /* since the items need much less than SVN_INT64_BUFFER_SIZE
4310 * bytes to represent their length, the split may not be optimal */
4312 right_count = revprops->sizes->nelts - left;
4314 /* if new_size is large, one side may exceed the pack size limit.
4315 * In that case, split before and after the modified revprop.*/
4316 if ( left_size > ffd->revprop_pack_size
4317 || right_size > ffd->revprop_pack_size)
4319 left_count = changed_index;
4320 right_count = revprops->sizes->nelts - left_count - 1;
4323 /* write the new, split files */
4326 SVN_ERR(repack_stream_open(&stream, fs, revprops, 0,
4327 left_count, files_to_delete, pool));
4328 SVN_ERR(repack_revprops(fs, revprops, 0, left_count,
4329 changed_index, serialized, new_total_size,
4333 if (left_count + right_count < revprops->sizes->nelts)
4335 SVN_ERR(repack_stream_open(&stream, fs, revprops, changed_index,
4336 changed_index + 1, files_to_delete,
4338 SVN_ERR(repack_revprops(fs, revprops, changed_index,
4340 changed_index, serialized, new_total_size,
4346 SVN_ERR(repack_stream_open(&stream, fs, revprops,
4347 revprops->sizes->nelts - right_count,
4348 revprops->sizes->nelts,
4349 files_to_delete, pool));
4350 SVN_ERR(repack_revprops(fs, revprops,
4351 revprops->sizes->nelts - right_count,
4352 revprops->sizes->nelts, changed_index,
4353 serialized, new_total_size, stream,
4357 /* write the new manifest */
4358 *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool);
4359 SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder,
4360 svn_io_file_del_none, pool, pool));
4362 for (i = 0; i < revprops->manifest->nelts; ++i)
4364 const char *filename = APR_ARRAY_IDX(revprops->manifest, i,
4366 SVN_ERR(svn_stream_printf(stream, pool, "%s\n", filename));
4369 SVN_ERR(svn_stream_close(stream));
4372 return SVN_NO_ERROR;
4375 /* Set the revision property list of revision REV in filesystem FS to
4376 PROPLIST. Use POOL for temporary allocations. */
4377 static svn_error_t *
4378 set_revision_proplist(svn_fs_t *fs,
4380 apr_hash_t *proplist,
4383 svn_boolean_t is_packed;
4384 svn_boolean_t bump_generation = FALSE;
4385 const char *final_path;
4386 const char *tmp_path;
4387 const char *perms_reference;
4388 apr_array_header_t *files_to_delete = NULL;
4390 SVN_ERR(ensure_revision_exists(fs, rev, pool));
4392 /* this info will not change while we hold the global FS write lock */
4393 is_packed = is_packed_revprop(fs, rev);
4395 /* Test whether revprops already exist for this revision.
4396 * Only then will we need to bump the revprop generation. */
4397 if (has_revprop_cache(fs, pool))
4401 bump_generation = TRUE;
4405 svn_node_kind_t kind;
4406 SVN_ERR(svn_io_check_path(path_revprops(fs, rev, pool), &kind,
4408 bump_generation = kind != svn_node_none;
4412 /* Serialize the new revprop data */
4414 SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete,
4415 fs, rev, proplist, pool));
4417 SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path,
4418 fs, rev, proplist, pool));
4420 /* We use the rev file of this revision as the perms reference,
4421 * because when setting revprops for the first time, the revprop
4422 * file won't exist and therefore can't serve as its own reference.
4423 * (Whereas the rev file should already exist at this point.)
4425 SVN_ERR(svn_fs_fs__path_rev_absolute(&perms_reference, fs, rev, pool));
4427 /* Now, switch to the new revprop data. */
4428 SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference,
4429 files_to_delete, bump_generation, pool));
4431 return SVN_NO_ERROR;
4435 svn_fs_fs__revision_proplist(apr_hash_t **proplist_p,
4440 SVN_ERR(get_revision_proplist(proplist_p, fs, rev, pool));
4442 return SVN_NO_ERROR;
4445 /* Represents where in the current svndiff data block each
4446 representation is. */
4450 /* The txdelta window cache to use or NULL. */
4451 svn_cache__t *window_cache;
4452 /* Caches un-deltified windows. May be NULL. */
4453 svn_cache__t *combined_cache;
4454 apr_off_t start; /* The starting offset for the raw
4455 svndiff/plaintext data minus header. */
4456 apr_off_t off; /* The current offset into the file. */
4457 apr_off_t end; /* The end offset of the raw data. */
4458 int ver; /* If a delta, what svndiff version? */
4462 /* See create_rep_state, which wraps this and adds another error. */
4463 static svn_error_t *
4464 create_rep_state_body(struct rep_state **rep_state,
4465 struct rep_args **rep_args,
4466 apr_file_t **file_hint,
4467 svn_revnum_t *rev_hint,
4468 representation_t *rep,
4472 fs_fs_data_t *ffd = fs->fsap_data;
4473 struct rep_state *rs = apr_pcalloc(pool, sizeof(*rs));
4474 struct rep_args *ra;
4475 unsigned char buf[4];
4479 * - refers to a valid revision,
4480 * - refers to a packed revision,
4481 * - as does the rep we want to read, and
4482 * - refers to the same pack file as the rep
4485 if ( file_hint && rev_hint && *file_hint
4486 && SVN_IS_VALID_REVNUM(*rev_hint)
4487 && *rev_hint < ffd->min_unpacked_rev
4488 && rep->revision < ffd->min_unpacked_rev
4489 && ( (*rev_hint / ffd->max_files_per_dir)
4490 == (rep->revision / ffd->max_files_per_dir)))
4492 /* ... we can re-use the same, already open file object
4495 SVN_ERR(get_packed_offset(&offset, fs, rep->revision, pool));
4497 offset += rep->offset;
4498 SVN_ERR(svn_io_file_seek(*file_hint, APR_SET, &offset, pool));
4500 rs->file = *file_hint;
4504 /* otherwise, create a new file object
4506 SVN_ERR(open_and_seek_representation(&rs->file, fs, rep, pool));
4509 /* remember the current file, if suggested by the caller */
4511 *file_hint = rs->file;
4513 *rev_hint = rep->revision;
4515 /* continue constructing RS and RA */
4516 rs->window_cache = ffd->txdelta_window_cache;
4517 rs->combined_cache = ffd->combined_window_cache;
4519 SVN_ERR(read_rep_line(&ra, rs->file, pool));
4520 SVN_ERR(get_file_offset(&rs->start, rs->file, pool));
4521 rs->off = rs->start;
4522 rs->end = rs->start + rep->size;
4527 /* This is a plaintext, so just return the current rep_state. */
4528 return SVN_NO_ERROR;
4530 /* We are dealing with a delta, find out what version. */
4531 SVN_ERR(svn_io_file_read_full2(rs->file, buf, sizeof(buf),
4533 /* ### Layering violation */
4534 if (! ((buf[0] == 'S') && (buf[1] == 'V') && (buf[2] == 'N')))
4535 return svn_error_create
4536 (SVN_ERR_FS_CORRUPT, NULL,
4537 _("Malformed svndiff data in representation"));
4539 rs->chunk_index = 0;
4542 return SVN_NO_ERROR;
4545 /* Read the rep args for REP in filesystem FS and create a rep_state
4546 for reading the representation. Return the rep_state in *REP_STATE
4547 and the rep args in *REP_ARGS, both allocated in POOL.
4549 When reading multiple reps, i.e. a skip delta chain, you may provide
4550 non-NULL FILE_HINT and REV_HINT. (If FILE_HINT is not NULL, in the first
4551 call it should be a pointer to NULL.) The function will use these variables
4552 to store the previous call results and tries to re-use them. This may
4553 result in significant savings in I/O for packed files.
4555 static svn_error_t *
4556 create_rep_state(struct rep_state **rep_state,
4557 struct rep_args **rep_args,
4558 apr_file_t **file_hint,
4559 svn_revnum_t *rev_hint,
4560 representation_t *rep,
4564 svn_error_t *err = create_rep_state_body(rep_state, rep_args,
4565 file_hint, rev_hint,
4567 if (err && err->apr_err == SVN_ERR_FS_CORRUPT)
4569 fs_fs_data_t *ffd = fs->fsap_data;
4571 /* ### This always returns "-1" for transaction reps, because
4572 ### this particular bit of code doesn't know if the rep is
4573 ### stored in the protorev or in the mutable area (for props
4574 ### or dir contents). It is pretty rare for FSFS to *read*
4575 ### from the protorev file, though, so this is probably OK.
4576 ### And anyone going to debug corruption errors is probably
4577 ### going to jump straight to this comment anyway! */
4578 return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
4579 "Corrupt representation '%s'",
4581 ? representation_string(rep, ffd->format, TRUE,
4585 /* ### Call representation_string() ? */
4586 return svn_error_trace(err);
4589 struct rep_read_baton
4591 /* The FS from which we're reading. */
4594 /* If not NULL, this is the base for the first delta window in rs_list */
4595 svn_stringbuf_t *base_window;
4597 /* The state of all prior delta representations. */
4598 apr_array_header_t *rs_list;
4600 /* The plaintext state, if there is a plaintext. */
4601 struct rep_state *src_state;
4603 /* The index of the current delta chunk, if we are reading a delta. */
4606 /* The buffer where we store undeltified data. */
4611 /* A checksum context for summing the data read in order to verify it.
4612 Note: we don't need to use the sha1 checksum because we're only doing
4613 data verification, for which md5 is perfectly safe. */
4614 svn_checksum_ctx_t *md5_checksum_ctx;
4616 svn_boolean_t checksum_finalized;
4618 /* The stored checksum of the representation we are reading, its
4619 length, and the amount we've read so far. Some of this
4620 information is redundant with rs_list and src_state, but it's
4621 convenient for the checksumming code to have it here. */
4622 svn_checksum_t *md5_checksum;
4627 /* The key for the fulltext cache for this rep, if there is a
4629 pair_cache_key_t fulltext_cache_key;
4630 /* The text we've been reading, if we're going to cache it. */
4631 svn_stringbuf_t *current_fulltext;
4633 /* Used for temporary allocations during the read. */
4636 /* Pool used to store file handles and other data that is persistant
4637 for the entire stream read. */
4638 apr_pool_t *filehandle_pool;
4641 /* Combine the name of the rev file in RS with the given OFFSET to form
4642 * a cache lookup key. Allocations will be made from POOL. May return
4643 * NULL if the key cannot be constructed. */
4645 get_window_key(struct rep_state *rs, apr_off_t offset, apr_pool_t *pool)
4648 const char *last_part;
4649 const char *name_last;
4651 /* the rev file name containing the txdelta window.
4652 * If this fails we are in serious trouble anyways.
4653 * And if nobody else detects the problems, the file content checksum
4654 * comparison _will_ find them.
4656 if (apr_file_name_get(&name, rs->file))
4659 /* Handle packed files as well by scanning backwards until we find the
4660 * revision or pack number. */
4661 name_last = name + strlen(name) - 1;
4662 while (! svn_ctype_isdigit(*name_last))
4665 last_part = name_last;
4666 while (svn_ctype_isdigit(*last_part))
4669 /* We must differentiate between packed files (as of today, the number
4670 * is being followed by a dot) and non-packed files (followed by \0).
4671 * Otherwise, there might be overlaps in the numbering range if the
4672 * repo gets packed after caching the txdeltas of non-packed revs.
4673 * => add the first non-digit char to the packed number. */
4674 if (name_last[1] != '\0')
4677 /* copy one char MORE than the actual number to mark packed files,
4678 * i.e. packed revision file content uses different key space then
4679 * non-packed ones: keys for packed rev file content ends with a dot
4680 * for non-packed rev files they end with a digit. */
4681 name = apr_pstrndup(pool, last_part + 1, name_last - last_part);
4682 return svn_fs_fs__combine_number_and_string(offset, name, pool);
4685 /* Read the WINDOW_P for the rep state RS from the current FSFS session's
4686 * cache. This will be a no-op and IS_CACHED will be set to FALSE if no
4687 * cache has been given. If a cache is available IS_CACHED will inform
4688 * the caller about the success of the lookup. Allocations (of the window
4689 * in particualar) will be made from POOL.
4691 * If the information could be found, put RS and the position within the
4692 * rev file into the same state as if the data had just been read from it.
4694 static svn_error_t *
4695 get_cached_window(svn_txdelta_window_t **window_p,
4696 struct rep_state *rs,
4697 svn_boolean_t *is_cached,
4700 if (! rs->window_cache)
4702 /* txdelta window has not been enabled */
4707 /* ask the cache for the desired txdelta window */
4708 svn_fs_fs__txdelta_cached_window_t *cached_window;
4709 SVN_ERR(svn_cache__get((void **) &cached_window,
4712 get_window_key(rs, rs->off, pool),
4717 /* found it. Pass it back to the caller. */
4718 *window_p = cached_window->window;
4720 /* manipulate the RS as if we just read the data */
4722 rs->off = cached_window->end_offset;
4724 /* manipulate the rev file as if we just read from it */
4725 SVN_ERR(svn_io_file_seek(rs->file, APR_SET, &rs->off, pool));
4729 return SVN_NO_ERROR;
4732 /* Store the WINDOW read at OFFSET for the rep state RS in the current
4733 * FSFS session's cache. This will be a no-op if no cache has been given.
4734 * Temporary allocations will be made from SCRATCH_POOL. */
4735 static svn_error_t *
4736 set_cached_window(svn_txdelta_window_t *window,
4737 struct rep_state *rs,
4739 apr_pool_t *scratch_pool)
4741 if (rs->window_cache)
4743 /* store the window and the first offset _past_ it */
4744 svn_fs_fs__txdelta_cached_window_t cached_window;
4746 cached_window.window = window;
4747 cached_window.end_offset = rs->off;
4749 /* but key it with the start offset because that is the known state
4750 * when we will look it up */
4751 return svn_cache__set(rs->window_cache,
4752 get_window_key(rs, offset, scratch_pool),
4757 return SVN_NO_ERROR;
4760 /* Read the WINDOW_P for the rep state RS from the current FSFS session's
4761 * cache. This will be a no-op and IS_CACHED will be set to FALSE if no
4762 * cache has been given. If a cache is available IS_CACHED will inform
4763 * the caller about the success of the lookup. Allocations (of the window
4764 * in particualar) will be made from POOL.
4766 static svn_error_t *
4767 get_cached_combined_window(svn_stringbuf_t **window_p,
4768 struct rep_state *rs,
4769 svn_boolean_t *is_cached,
4772 if (! rs->combined_cache)
4774 /* txdelta window has not been enabled */
4779 /* ask the cache for the desired txdelta window */
4780 return svn_cache__get((void **)window_p,
4783 get_window_key(rs, rs->start, pool),
4787 return SVN_NO_ERROR;
4790 /* Store the WINDOW read at OFFSET for the rep state RS in the current
4791 * FSFS session's cache. This will be a no-op if no cache has been given.
4792 * Temporary allocations will be made from SCRATCH_POOL. */
4793 static svn_error_t *
4794 set_cached_combined_window(svn_stringbuf_t *window,
4795 struct rep_state *rs,
4797 apr_pool_t *scratch_pool)
4799 if (rs->combined_cache)
4801 /* but key it with the start offset because that is the known state
4802 * when we will look it up */
4803 return svn_cache__set(rs->combined_cache,
4804 get_window_key(rs, offset, scratch_pool),
4809 return SVN_NO_ERROR;
4812 /* Build an array of rep_state structures in *LIST giving the delta
4813 reps from first_rep to a plain-text or self-compressed rep. Set
4814 *SRC_STATE to the plain-text rep we find at the end of the chain,
4815 or to NULL if the final delta representation is self-compressed.
4816 The representation to start from is designated by filesystem FS, id
4817 ID, and representation REP.
4818 Also, set *WINDOW_P to the base window content for *LIST, if it
4819 could be found in cache. Otherwise, *LIST will contain the base
4820 representation for the whole delta chain.
4821 Finally, return the expanded size of the representation in
4822 *EXPANDED_SIZE. It will take care of cases where only the on-disk
4824 static svn_error_t *
4825 build_rep_list(apr_array_header_t **list,
4826 svn_stringbuf_t **window_p,
4827 struct rep_state **src_state,
4828 svn_filesize_t *expanded_size,
4830 representation_t *first_rep,
4833 representation_t rep;
4834 struct rep_state *rs = NULL;
4835 struct rep_args *rep_args;
4836 svn_boolean_t is_cached = FALSE;
4837 apr_file_t *last_file = NULL;
4838 svn_revnum_t last_revision;
4840 *list = apr_array_make(pool, 1, sizeof(struct rep_state *));
4843 /* The value as stored in the data struct.
4844 0 is either for unknown length or actually zero length. */
4845 *expanded_size = first_rep->expanded_size;
4847 /* for the top-level rep, we need the rep_args */
4848 SVN_ERR(create_rep_state(&rs, &rep_args, &last_file,
4849 &last_revision, &rep, fs, pool));
4851 /* Unknown size or empty representation?
4852 That implies the this being the first iteration.
4853 Usually size equals on-disk size, except for empty,
4854 compressed representations (delta, size = 4).
4855 Please note that for all non-empty deltas have
4856 a 4-byte header _plus_ some data. */
4857 if (*expanded_size == 0)
4858 if (! rep_args->is_delta || first_rep->size != 4)
4859 *expanded_size = first_rep->size;
4863 /* fetch state, if that has not been done already */
4865 SVN_ERR(create_rep_state(&rs, &rep_args, &last_file,
4866 &last_revision, &rep, fs, pool));
4868 SVN_ERR(get_cached_combined_window(window_p, rs, &is_cached, pool));
4871 /* We already have a reconstructed window in our cache.
4872 Write a pseudo rep_state with the full length. */
4873 rs->off = rs->start;
4874 rs->end = rs->start + (*window_p)->len;
4876 return SVN_NO_ERROR;
4879 if (!rep_args->is_delta)
4881 /* This is a plaintext, so just return the current rep_state. */
4883 return SVN_NO_ERROR;
4886 /* Push this rep onto the list. If it's self-compressed, we're done. */
4887 APR_ARRAY_PUSH(*list, struct rep_state *) = rs;
4888 if (rep_args->is_delta_vs_empty)
4891 return SVN_NO_ERROR;
4894 rep.revision = rep_args->base_revision;
4895 rep.offset = rep_args->base_offset;
4896 rep.size = rep_args->base_length;
4904 /* Create a rep_read_baton structure for node revision NODEREV in
4905 filesystem FS and store it in *RB_P. If FULLTEXT_CACHE_KEY is not
4906 NULL, it is the rep's key in the fulltext cache, and a stringbuf
4907 must be allocated to store the text. Perform all allocations in
4908 POOL. If rep is mutable, it must be for file contents. */
4909 static svn_error_t *
4910 rep_read_get_baton(struct rep_read_baton **rb_p,
4912 representation_t *rep,
4913 pair_cache_key_t fulltext_cache_key,
4916 struct rep_read_baton *b;
4918 b = apr_pcalloc(pool, sizeof(*b));
4920 b->base_window = NULL;
4923 b->md5_checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool);
4924 b->checksum_finalized = FALSE;
4925 b->md5_checksum = svn_checksum_dup(rep->md5_checksum, pool);
4926 b->len = rep->expanded_size;
4928 b->fulltext_cache_key = fulltext_cache_key;
4929 b->pool = svn_pool_create(pool);
4930 b->filehandle_pool = svn_pool_create(pool);
4932 SVN_ERR(build_rep_list(&b->rs_list, &b->base_window,
4933 &b->src_state, &b->len, fs, rep,
4934 b->filehandle_pool));
4936 if (SVN_IS_VALID_REVNUM(fulltext_cache_key.revision))
4937 b->current_fulltext = svn_stringbuf_create_ensure
4938 ((apr_size_t)b->len,
4939 b->filehandle_pool);
4941 b->current_fulltext = NULL;
4943 /* Save our output baton. */
4946 return SVN_NO_ERROR;
4949 /* Skip forwards to THIS_CHUNK in REP_STATE and then read the next delta
4950 window into *NWIN. */
4951 static svn_error_t *
4952 read_delta_window(svn_txdelta_window_t **nwin, int this_chunk,
4953 struct rep_state *rs, apr_pool_t *pool)
4955 svn_stream_t *stream;
4956 svn_boolean_t is_cached;
4957 apr_off_t old_offset;
4959 SVN_ERR_ASSERT(rs->chunk_index <= this_chunk);
4961 /* RS->FILE may be shared between RS instances -> make sure we point
4962 * to the right data. */
4963 SVN_ERR(svn_io_file_seek(rs->file, APR_SET, &rs->off, pool));
4965 /* Skip windows to reach the current chunk if we aren't there yet. */
4966 while (rs->chunk_index < this_chunk)
4968 SVN_ERR(svn_txdelta_skip_svndiff_window(rs->file, rs->ver, pool));
4970 SVN_ERR(get_file_offset(&rs->off, rs->file, pool));
4971 if (rs->off >= rs->end)
4972 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
4973 _("Reading one svndiff window read "
4974 "beyond the end of the "
4978 /* Read the next window. But first, try to find it in the cache. */
4979 SVN_ERR(get_cached_window(nwin, rs, &is_cached, pool));
4981 return SVN_NO_ERROR;
4983 /* Actually read the next window. */
4984 old_offset = rs->off;
4985 stream = svn_stream_from_aprfile2(rs->file, TRUE, pool);
4986 SVN_ERR(svn_txdelta_read_svndiff_window(nwin, stream, rs->ver, pool));
4988 SVN_ERR(get_file_offset(&rs->off, rs->file, pool));
4990 if (rs->off > rs->end)
4991 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
4992 _("Reading one svndiff window read beyond "
4993 "the end of the representation"));
4995 /* the window has not been cached before, thus cache it now
4996 * (if caching is used for them at all) */
4997 return set_cached_window(*nwin, rs, old_offset, pool);
5000 /* Read SIZE bytes from the representation RS and return it in *NWIN. */
5001 static svn_error_t *
5002 read_plain_window(svn_stringbuf_t **nwin, struct rep_state *rs,
5003 apr_size_t size, apr_pool_t *pool)
5005 /* RS->FILE may be shared between RS instances -> make sure we point
5006 * to the right data. */
5007 SVN_ERR(svn_io_file_seek(rs->file, APR_SET, &rs->off, pool));
5009 /* Read the plain data. */
5010 *nwin = svn_stringbuf_create_ensure(size, pool);
5011 SVN_ERR(svn_io_file_read_full2(rs->file, (*nwin)->data, size, NULL, NULL,
5013 (*nwin)->data[size] = 0;
5016 rs->off += (apr_off_t)size;
5018 return SVN_NO_ERROR;
5021 /* Get the undeltified window that is a result of combining all deltas
5022 from the current desired representation identified in *RB with its
5023 base representation. Store the window in *RESULT. */
5024 static svn_error_t *
5025 get_combined_window(svn_stringbuf_t **result,
5026 struct rep_read_baton *rb)
5028 apr_pool_t *pool, *new_pool, *window_pool;
5030 svn_txdelta_window_t *window;
5031 apr_array_header_t *windows;
5032 svn_stringbuf_t *source, *buf = rb->base_window;
5033 struct rep_state *rs;
5035 /* Read all windows that we need to combine. This is fine because
5036 the size of each window is relatively small (100kB) and skip-
5037 delta limits the number of deltas in a chain to well under 100.
5038 Stop early if one of them does not depend on its predecessors. */
5039 window_pool = svn_pool_create(rb->pool);
5040 windows = apr_array_make(window_pool, 0, sizeof(svn_txdelta_window_t *));
5041 for (i = 0; i < rb->rs_list->nelts; ++i)
5043 rs = APR_ARRAY_IDX(rb->rs_list, i, struct rep_state *);
5044 SVN_ERR(read_delta_window(&window, rb->chunk_index, rs, window_pool));
5046 APR_ARRAY_PUSH(windows, svn_txdelta_window_t *) = window;
5047 if (window->src_ops == 0)
5054 /* Combine in the windows from the other delta reps. */
5055 pool = svn_pool_create(rb->pool);
5056 for (--i; i >= 0; --i)
5059 rs = APR_ARRAY_IDX(rb->rs_list, i, struct rep_state *);
5060 window = APR_ARRAY_IDX(windows, i, svn_txdelta_window_t *);
5062 /* Maybe, we've got a PLAIN start representation. If we do, read
5063 as much data from it as the needed for the txdelta window's source
5065 Note that BUF / SOURCE may only be NULL in the first iteration. */
5067 if (source == NULL && rb->src_state != NULL)
5068 SVN_ERR(read_plain_window(&source, rb->src_state, window->sview_len,
5071 /* Combine this window with the current one. */
5072 new_pool = svn_pool_create(rb->pool);
5073 buf = svn_stringbuf_create_ensure(window->tview_len, new_pool);
5074 buf->len = window->tview_len;
5076 svn_txdelta_apply_instructions(window, source ? source->data : NULL,
5077 buf->data, &buf->len);
5078 if (buf->len != window->tview_len)
5079 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
5080 _("svndiff window length is "
5083 /* Cache windows only if the whole rep content could be read as a
5084 single chunk. Only then will no other chunk need a deeper RS
5085 list than the cached chunk. */
5086 if ((rb->chunk_index == 0) && (rs->off == rs->end))
5087 SVN_ERR(set_cached_combined_window(buf, rs, rs->start, new_pool));
5089 /* Cycle pools so that we only need to hold three windows at a time. */
5090 svn_pool_destroy(pool);
5094 svn_pool_destroy(window_pool);
5097 return SVN_NO_ERROR;
5100 /* Returns whether or not the expanded fulltext of the file is cachable
5101 * based on its size SIZE. The decision depends on the cache used by RB.
5103 static svn_boolean_t
5104 fulltext_size_is_cachable(fs_fs_data_t *ffd, svn_filesize_t size)
5106 return (size < APR_SIZE_MAX)
5107 && svn_cache__is_cachable(ffd->fulltext_cache, (apr_size_t)size);
5110 /* Close method used on streams returned by read_representation().
5112 static svn_error_t *
5113 rep_read_contents_close(void *baton)
5115 struct rep_read_baton *rb = baton;
5117 svn_pool_destroy(rb->pool);
5118 svn_pool_destroy(rb->filehandle_pool);
5120 return SVN_NO_ERROR;
5123 /* Return the next *LEN bytes of the rep and store them in *BUF. */
5124 static svn_error_t *
5125 get_contents(struct rep_read_baton *rb,
5129 apr_size_t copy_len, remaining = *len;
5131 struct rep_state *rs;
5133 /* Special case for when there are no delta reps, only a plain
5135 if (rb->rs_list->nelts == 0)
5137 copy_len = remaining;
5140 if (rb->base_window != NULL)
5142 /* We got the desired rep directly from the cache.
5143 This is where we need the pseudo rep_state created
5144 by build_rep_list(). */
5145 apr_size_t offset = (apr_size_t)(rs->off - rs->start);
5146 if (copy_len + offset > rb->base_window->len)
5147 copy_len = offset < rb->base_window->len
5148 ? rb->base_window->len - offset
5151 memcpy (cur, rb->base_window->data + offset, copy_len);
5155 if (((apr_off_t) copy_len) > rs->end - rs->off)
5156 copy_len = (apr_size_t) (rs->end - rs->off);
5157 SVN_ERR(svn_io_file_read_full2(rs->file, cur, copy_len, NULL,
5161 rs->off += copy_len;
5163 return SVN_NO_ERROR;
5166 while (remaining > 0)
5168 /* If we have buffered data from a previous chunk, use that. */
5171 /* Determine how much to copy from the buffer. */
5172 copy_len = rb->buf_len - rb->buf_pos;
5173 if (copy_len > remaining)
5174 copy_len = remaining;
5176 /* Actually copy the data. */
5177 memcpy(cur, rb->buf + rb->buf_pos, copy_len);
5178 rb->buf_pos += copy_len;
5180 remaining -= copy_len;
5182 /* If the buffer is all used up, clear it and empty the
5184 if (rb->buf_pos == rb->buf_len)
5186 svn_pool_clear(rb->pool);
5192 svn_stringbuf_t *sbuf = NULL;
5194 rs = APR_ARRAY_IDX(rb->rs_list, 0, struct rep_state *);
5195 if (rs->off == rs->end)
5198 /* Get more buffered data by evaluating a chunk. */
5199 SVN_ERR(get_combined_window(&sbuf, rb));
5202 rb->buf_len = sbuf->len;
5203 rb->buf = sbuf->data;
5210 return SVN_NO_ERROR;
5213 /* BATON is of type `rep_read_baton'; read the next *LEN bytes of the
5214 representation and store them in *BUF. Sum as we read and verify
5215 the MD5 sum at the end. */
5216 static svn_error_t *
5217 rep_read_contents(void *baton,
5221 struct rep_read_baton *rb = baton;
5223 /* Get the next block of data. */
5224 SVN_ERR(get_contents(rb, buf, len));
5226 if (rb->current_fulltext)
5227 svn_stringbuf_appendbytes(rb->current_fulltext, buf, *len);
5229 /* Perform checksumming. We want to check the checksum as soon as
5230 the last byte of data is read, in case the caller never performs
5231 a short read, but we don't want to finalize the MD5 context
5233 if (!rb->checksum_finalized)
5235 SVN_ERR(svn_checksum_update(rb->md5_checksum_ctx, buf, *len));
5237 if (rb->off == rb->len)
5239 svn_checksum_t *md5_checksum;
5241 rb->checksum_finalized = TRUE;
5242 SVN_ERR(svn_checksum_final(&md5_checksum, rb->md5_checksum_ctx,
5244 if (!svn_checksum_match(md5_checksum, rb->md5_checksum))
5245 return svn_error_create(SVN_ERR_FS_CORRUPT,
5246 svn_checksum_mismatch_err(rb->md5_checksum, md5_checksum,
5248 _("Checksum mismatch while reading representation")),
5253 if (rb->off == rb->len && rb->current_fulltext)
5255 fs_fs_data_t *ffd = rb->fs->fsap_data;
5256 SVN_ERR(svn_cache__set(ffd->fulltext_cache, &rb->fulltext_cache_key,
5257 rb->current_fulltext, rb->pool));
5258 rb->current_fulltext = NULL;
5261 return SVN_NO_ERROR;
5265 /* Return a stream in *CONTENTS_P that will read the contents of a
5266 representation stored at the location given by REP. Appropriate
5267 for any kind of immutable representation, but only for file
5268 contents (not props or directory contents) in mutable
5271 If REP is NULL, the representation is assumed to be empty, and the
5272 empty stream is returned.
5274 static svn_error_t *
5275 read_representation(svn_stream_t **contents_p,
5277 representation_t *rep,
5282 *contents_p = svn_stream_empty(pool);
5286 fs_fs_data_t *ffd = fs->fsap_data;
5287 pair_cache_key_t fulltext_cache_key = { 0 };
5288 svn_filesize_t len = rep->expanded_size ? rep->expanded_size : rep->size;
5289 struct rep_read_baton *rb;
5291 fulltext_cache_key.revision = rep->revision;
5292 fulltext_cache_key.second = rep->offset;
5293 if (ffd->fulltext_cache && SVN_IS_VALID_REVNUM(rep->revision)
5294 && fulltext_size_is_cachable(ffd, len))
5296 svn_stringbuf_t *fulltext;
5297 svn_boolean_t is_cached;
5298 SVN_ERR(svn_cache__get((void **) &fulltext, &is_cached,
5299 ffd->fulltext_cache, &fulltext_cache_key,
5303 *contents_p = svn_stream_from_stringbuf(fulltext, pool);
5304 return SVN_NO_ERROR;
5308 fulltext_cache_key.revision = SVN_INVALID_REVNUM;
5310 SVN_ERR(rep_read_get_baton(&rb, fs, rep, fulltext_cache_key, pool));
5312 *contents_p = svn_stream_create(rb, pool);
5313 svn_stream_set_read(*contents_p, rep_read_contents);
5314 svn_stream_set_close(*contents_p, rep_read_contents_close);
5317 return SVN_NO_ERROR;
5321 svn_fs_fs__get_contents(svn_stream_t **contents_p,
5323 node_revision_t *noderev,
5326 return read_representation(contents_p, fs, noderev->data_rep, pool);
5329 /* Baton used when reading delta windows. */
5330 struct delta_read_baton
5332 struct rep_state *rs;
5333 svn_checksum_t *checksum;
5336 /* This implements the svn_txdelta_next_window_fn_t interface. */
5337 static svn_error_t *
5338 delta_read_next_window(svn_txdelta_window_t **window, void *baton,
5341 struct delta_read_baton *drb = baton;
5343 if (drb->rs->off == drb->rs->end)
5346 return SVN_NO_ERROR;
5349 return read_delta_window(window, drb->rs->chunk_index, drb->rs, pool);
5352 /* This implements the svn_txdelta_md5_digest_fn_t interface. */
5353 static const unsigned char *
5354 delta_read_md5_digest(void *baton)
5356 struct delta_read_baton *drb = baton;
5358 if (drb->checksum->kind == svn_checksum_md5)
5359 return drb->checksum->digest;
5365 svn_fs_fs__get_file_delta_stream(svn_txdelta_stream_t **stream_p,
5367 node_revision_t *source,
5368 node_revision_t *target,
5371 svn_stream_t *source_stream, *target_stream;
5373 /* Try a shortcut: if the target is stored as a delta against the source,
5374 then just use that delta. */
5375 if (source && source->data_rep && target->data_rep)
5377 struct rep_state *rep_state;
5378 struct rep_args *rep_args;
5380 /* Read target's base rep if any. */
5381 SVN_ERR(create_rep_state(&rep_state, &rep_args, NULL, NULL,
5382 target->data_rep, fs, pool));
5383 /* If that matches source, then use this delta as is. */
5384 if (rep_args->is_delta
5385 && (rep_args->is_delta_vs_empty
5386 || (rep_args->base_revision == source->data_rep->revision
5387 && rep_args->base_offset == source->data_rep->offset)))
5389 /* Create the delta read baton. */
5390 struct delta_read_baton *drb = apr_pcalloc(pool, sizeof(*drb));
5391 drb->rs = rep_state;
5392 drb->checksum = svn_checksum_dup(target->data_rep->md5_checksum,
5394 *stream_p = svn_txdelta_stream_create(drb, delta_read_next_window,
5395 delta_read_md5_digest, pool);
5396 return SVN_NO_ERROR;
5399 SVN_ERR(svn_io_file_close(rep_state->file, pool));
5402 /* Read both fulltexts and construct a delta. */
5404 SVN_ERR(read_representation(&source_stream, fs, source->data_rep, pool));
5406 source_stream = svn_stream_empty(pool);
5407 SVN_ERR(read_representation(&target_stream, fs, target->data_rep, pool));
5409 /* Because source and target stream will already verify their content,
5410 * there is no need to do this once more. In particular if the stream
5411 * content is being fetched from cache. */
5412 svn_txdelta2(stream_p, source_stream, target_stream, FALSE, pool);
5414 return SVN_NO_ERROR;
5417 /* Baton for cache_access_wrapper. Wraps the original parameters of
5418 * svn_fs_fs__try_process_file_content().
5420 typedef struct cache_access_wrapper_baton_t
5422 svn_fs_process_contents_func_t func;
5424 } cache_access_wrapper_baton_t;
5426 /* Wrapper to translate between svn_fs_process_contents_func_t and
5427 * svn_cache__partial_getter_func_t.
5429 static svn_error_t *
5430 cache_access_wrapper(void **out,
5432 apr_size_t data_len,
5436 cache_access_wrapper_baton_t *wrapper_baton = baton;
5438 SVN_ERR(wrapper_baton->func((const unsigned char *)data,
5439 data_len - 1, /* cache adds terminating 0 */
5440 wrapper_baton->baton,
5443 /* non-NULL value to signal the calling cache that all went well */
5446 return SVN_NO_ERROR;
5450 svn_fs_fs__try_process_file_contents(svn_boolean_t *success,
5452 node_revision_t *noderev,
5453 svn_fs_process_contents_func_t processor,
5457 representation_t *rep = noderev->data_rep;
5460 fs_fs_data_t *ffd = fs->fsap_data;
5461 pair_cache_key_t fulltext_cache_key = { 0 };
5463 fulltext_cache_key.revision = rep->revision;
5464 fulltext_cache_key.second = rep->offset;
5465 if (ffd->fulltext_cache && SVN_IS_VALID_REVNUM(rep->revision)
5466 && fulltext_size_is_cachable(ffd, rep->expanded_size))
5468 cache_access_wrapper_baton_t wrapper_baton;
5471 wrapper_baton.func = processor;
5472 wrapper_baton.baton = baton;
5473 return svn_cache__get_partial(&dummy, success,
5474 ffd->fulltext_cache,
5475 &fulltext_cache_key,
5476 cache_access_wrapper,
5483 return SVN_NO_ERROR;
5486 /* Fetch the contents of a directory into ENTRIES. Values are stored
5487 as filename to string mappings; further conversion is necessary to
5488 convert them into svn_fs_dirent_t values. */
5489 static svn_error_t *
5490 get_dir_contents(apr_hash_t *entries,
5492 node_revision_t *noderev,
5495 svn_stream_t *contents;
5497 if (noderev->data_rep && noderev->data_rep->txn_id)
5499 const char *filename = path_txn_node_children(fs, noderev->id, pool);
5501 /* The representation is mutable. Read the old directory
5502 contents from the mutable children file, followed by the
5503 changes we've made in this transaction. */
5504 SVN_ERR(svn_stream_open_readonly(&contents, filename, pool, pool));
5505 SVN_ERR(svn_hash_read2(entries, contents, SVN_HASH_TERMINATOR, pool));
5506 SVN_ERR(svn_hash_read_incremental(entries, contents, NULL, pool));
5507 SVN_ERR(svn_stream_close(contents));
5509 else if (noderev->data_rep)
5511 /* use a temporary pool for temp objects.
5512 * Also undeltify content before parsing it. Otherwise, we could only
5513 * parse it byte-by-byte.
5515 apr_pool_t *text_pool = svn_pool_create(pool);
5516 apr_size_t len = noderev->data_rep->expanded_size
5517 ? (apr_size_t)noderev->data_rep->expanded_size
5518 : (apr_size_t)noderev->data_rep->size;
5519 svn_stringbuf_t *text = svn_stringbuf_create_ensure(len, text_pool);
5522 /* The representation is immutable. Read it normally. */
5523 SVN_ERR(read_representation(&contents, fs, noderev->data_rep, text_pool));
5524 SVN_ERR(svn_stream_read(contents, text->data, &text->len));
5525 SVN_ERR(svn_stream_close(contents));
5527 /* de-serialize hash */
5528 contents = svn_stream_from_stringbuf(text, text_pool);
5529 SVN_ERR(svn_hash_read2(entries, contents, SVN_HASH_TERMINATOR, pool));
5531 svn_pool_destroy(text_pool);
5534 return SVN_NO_ERROR;
5539 unparse_dir_entry(svn_node_kind_t kind, const svn_fs_id_t *id,
5542 return apr_psprintf(pool, "%s %s",
5543 (kind == svn_node_file) ? KIND_FILE : KIND_DIR,
5544 svn_fs_fs__id_unparse(id, pool)->data);
5547 /* Given a hash ENTRIES of dirent structions, return a hash in
5548 *STR_ENTRIES_P, that has svn_string_t as the values in the format
5549 specified by the fs_fs directory contents file. Perform
5550 allocations in POOL. */
5551 static svn_error_t *
5552 unparse_dir_entries(apr_hash_t **str_entries_p,
5553 apr_hash_t *entries,
5556 apr_hash_index_t *hi;
5558 /* For now, we use a our own hash function to ensure that we get a
5559 * (largely) stable order when serializing the data. It also gives
5560 * us some performance improvement.
5563 * Use some sorted or other fixed order data container.
5565 *str_entries_p = svn_hash__make(pool);
5567 for (hi = apr_hash_first(pool, entries); hi; hi = apr_hash_next(hi))
5571 svn_fs_dirent_t *dirent = svn__apr_hash_index_val(hi);
5572 const char *new_val;
5574 apr_hash_this(hi, &key, &klen, NULL);
5575 new_val = unparse_dir_entry(dirent->kind, dirent->id, pool);
5576 apr_hash_set(*str_entries_p, key, klen,
5577 svn_string_create(new_val, pool));
5580 return SVN_NO_ERROR;
5584 /* Given a hash STR_ENTRIES with values as svn_string_t as specified
5585 in an FSFS directory contents listing, return a hash of dirents in
5586 *ENTRIES_P. Perform allocations in POOL. */
5587 static svn_error_t *
5588 parse_dir_entries(apr_hash_t **entries_p,
5589 apr_hash_t *str_entries,
5590 const char *unparsed_id,
5593 apr_hash_index_t *hi;
5595 *entries_p = apr_hash_make(pool);
5597 /* Translate the string dir entries into real entries. */
5598 for (hi = apr_hash_first(pool, str_entries); hi; hi = apr_hash_next(hi))
5600 const char *name = svn__apr_hash_index_key(hi);
5601 svn_string_t *str_val = svn__apr_hash_index_val(hi);
5602 char *str, *last_str;
5603 svn_fs_dirent_t *dirent = apr_pcalloc(pool, sizeof(*dirent));
5605 last_str = apr_pstrdup(pool, str_val->data);
5606 dirent->name = apr_pstrdup(pool, name);
5608 str = svn_cstring_tokenize(" ", &last_str);
5610 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
5611 _("Directory entry corrupt in '%s'"),
5614 if (strcmp(str, KIND_FILE) == 0)
5616 dirent->kind = svn_node_file;
5618 else if (strcmp(str, KIND_DIR) == 0)
5620 dirent->kind = svn_node_dir;
5624 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
5625 _("Directory entry corrupt in '%s'"),
5629 str = svn_cstring_tokenize(" ", &last_str);
5631 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
5632 _("Directory entry corrupt in '%s'"),
5635 dirent->id = svn_fs_fs__id_parse(str, strlen(str), pool);
5637 svn_hash_sets(*entries_p, dirent->name, dirent);
5640 return SVN_NO_ERROR;
5643 /* Return the cache object in FS responsible to storing the directory
5644 * the NODEREV. If none exists, return NULL. */
5645 static svn_cache__t *
5646 locate_dir_cache(svn_fs_t *fs,
5647 node_revision_t *noderev)
5649 fs_fs_data_t *ffd = fs->fsap_data;
5650 return svn_fs_fs__id_txn_id(noderev->id)
5651 ? ffd->txn_dir_cache
5656 svn_fs_fs__rep_contents_dir(apr_hash_t **entries_p,
5658 node_revision_t *noderev,
5661 const char *unparsed_id = NULL;
5662 apr_hash_t *unparsed_entries, *parsed_entries;
5664 /* find the cache we may use */
5665 svn_cache__t *cache = locate_dir_cache(fs, noderev);
5668 svn_boolean_t found;
5670 unparsed_id = svn_fs_fs__id_unparse(noderev->id, pool)->data;
5671 SVN_ERR(svn_cache__get((void **) entries_p, &found, cache,
5672 unparsed_id, pool));
5674 return SVN_NO_ERROR;
5677 /* Read in the directory hash. */
5678 unparsed_entries = apr_hash_make(pool);
5679 SVN_ERR(get_dir_contents(unparsed_entries, fs, noderev, pool));
5680 SVN_ERR(parse_dir_entries(&parsed_entries, unparsed_entries,
5681 unparsed_id, pool));
5683 /* Update the cache, if we are to use one. */
5685 SVN_ERR(svn_cache__set(cache, unparsed_id, parsed_entries, pool));
5687 *entries_p = parsed_entries;
5688 return SVN_NO_ERROR;
5692 svn_fs_fs__rep_contents_dir_entry(svn_fs_dirent_t **dirent,
5694 node_revision_t *noderev,
5696 apr_pool_t *result_pool,
5697 apr_pool_t *scratch_pool)
5699 svn_boolean_t found = FALSE;
5701 /* find the cache we may use */
5702 svn_cache__t *cache = locate_dir_cache(fs, noderev);
5705 const char *unparsed_id =
5706 svn_fs_fs__id_unparse(noderev->id, scratch_pool)->data;
5709 SVN_ERR(svn_cache__get_partial((void **)dirent,
5713 svn_fs_fs__extract_dir_entry,
5718 /* fetch data from disk if we did not find it in the cache */
5721 apr_hash_t *entries;
5722 svn_fs_dirent_t *entry;
5723 svn_fs_dirent_t *entry_copy = NULL;
5725 /* read the dir from the file system. It will probably be put it
5726 into the cache for faster lookup in future calls. */
5727 SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, noderev,
5730 /* find desired entry and return a copy in POOL, if found */
5731 entry = svn_hash_gets(entries, name);
5734 entry_copy = apr_palloc(result_pool, sizeof(*entry_copy));
5735 entry_copy->name = apr_pstrdup(result_pool, entry->name);
5736 entry_copy->id = svn_fs_fs__id_copy(entry->id, result_pool);
5737 entry_copy->kind = entry->kind;
5740 *dirent = entry_copy;
5743 return SVN_NO_ERROR;
5747 svn_fs_fs__get_proplist(apr_hash_t **proplist_p,
5749 node_revision_t *noderev,
5752 apr_hash_t *proplist;
5753 svn_stream_t *stream;
5755 if (noderev->prop_rep && noderev->prop_rep->txn_id)
5757 const char *filename = path_txn_node_props(fs, noderev->id, pool);
5758 proplist = apr_hash_make(pool);
5760 SVN_ERR(svn_stream_open_readonly(&stream, filename, pool, pool));
5761 SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool));
5762 SVN_ERR(svn_stream_close(stream));
5764 else if (noderev->prop_rep)
5766 fs_fs_data_t *ffd = fs->fsap_data;
5767 representation_t *rep = noderev->prop_rep;
5768 pair_cache_key_t key = { 0 };
5770 key.revision = rep->revision;
5771 key.second = rep->offset;
5772 if (ffd->properties_cache && SVN_IS_VALID_REVNUM(rep->revision))
5774 svn_boolean_t is_cached;
5775 SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached,
5776 ffd->properties_cache, &key, pool));
5778 return SVN_NO_ERROR;
5781 proplist = apr_hash_make(pool);
5782 SVN_ERR(read_representation(&stream, fs, noderev->prop_rep, pool));
5783 SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool));
5784 SVN_ERR(svn_stream_close(stream));
5786 if (ffd->properties_cache && SVN_IS_VALID_REVNUM(rep->revision))
5787 SVN_ERR(svn_cache__set(ffd->properties_cache, &key, proplist, pool));
5791 /* return an empty prop list if the node doesn't have any props */
5792 proplist = apr_hash_make(pool);
5795 *proplist_p = proplist;
5797 return SVN_NO_ERROR;
5801 svn_fs_fs__file_length(svn_filesize_t *length,
5802 node_revision_t *noderev,
5805 if (noderev->data_rep)
5806 *length = noderev->data_rep->expanded_size;
5810 return SVN_NO_ERROR;
5814 svn_fs_fs__noderev_same_rep_key(representation_t *a,
5815 representation_t *b)
5820 if (a == NULL || b == NULL)
5823 if (a->offset != b->offset)
5826 if (a->revision != b->revision)
5829 if (a->uniquifier == b->uniquifier)
5832 if (a->uniquifier == NULL || b->uniquifier == NULL)
5835 return strcmp(a->uniquifier, b->uniquifier) == 0;
5839 svn_fs_fs__file_checksum(svn_checksum_t **checksum,
5840 node_revision_t *noderev,
5841 svn_checksum_kind_t kind,
5844 if (noderev->data_rep)
5848 case svn_checksum_md5:
5849 *checksum = svn_checksum_dup(noderev->data_rep->md5_checksum,
5852 case svn_checksum_sha1:
5853 *checksum = svn_checksum_dup(noderev->data_rep->sha1_checksum,
5863 return SVN_NO_ERROR;
5867 svn_fs_fs__rep_copy(representation_t *rep,
5870 representation_t *rep_new;
5875 rep_new = apr_pcalloc(pool, sizeof(*rep_new));
5877 memcpy(rep_new, rep, sizeof(*rep_new));
5878 rep_new->md5_checksum = svn_checksum_dup(rep->md5_checksum, pool);
5879 rep_new->sha1_checksum = svn_checksum_dup(rep->sha1_checksum, pool);
5880 rep_new->uniquifier = apr_pstrdup(pool, rep->uniquifier);
5885 /* Merge the internal-use-only CHANGE into a hash of public-FS
5886 svn_fs_path_change2_t CHANGES, collapsing multiple changes into a
5887 single summarical (is that real word?) change per path. Also keep
5888 the COPYFROM_CACHE up to date with new adds and replaces. */
5889 static svn_error_t *
5890 fold_change(apr_hash_t *changes,
5891 const change_t *change,
5892 apr_hash_t *copyfrom_cache)
5894 apr_pool_t *pool = apr_hash_pool_get(changes);
5895 svn_fs_path_change2_t *old_change, *new_change;
5897 apr_size_t path_len = strlen(change->path);
5899 if ((old_change = apr_hash_get(changes, change->path, path_len)))
5901 /* This path already exists in the hash, so we have to merge
5902 this change into the already existing one. */
5904 /* Sanity check: only allow NULL node revision ID in the
5906 if ((! change->noderev_id) && (change->kind != svn_fs_path_change_reset))
5907 return svn_error_create
5908 (SVN_ERR_FS_CORRUPT, NULL,
5909 _("Missing required node revision ID"));
5911 /* Sanity check: we should be talking about the same node
5912 revision ID as our last change except where the last change
5914 if (change->noderev_id
5915 && (! svn_fs_fs__id_eq(old_change->node_rev_id, change->noderev_id))
5916 && (old_change->change_kind != svn_fs_path_change_delete))
5917 return svn_error_create
5918 (SVN_ERR_FS_CORRUPT, NULL,
5919 _("Invalid change ordering: new node revision ID "
5922 /* Sanity check: an add, replacement, or reset must be the first
5923 thing to follow a deletion. */
5924 if ((old_change->change_kind == svn_fs_path_change_delete)
5925 && (! ((change->kind == svn_fs_path_change_replace)
5926 || (change->kind == svn_fs_path_change_reset)
5927 || (change->kind == svn_fs_path_change_add))))
5928 return svn_error_create
5929 (SVN_ERR_FS_CORRUPT, NULL,
5930 _("Invalid change ordering: non-add change on deleted path"));
5932 /* Sanity check: an add can't follow anything except
5933 a delete or reset. */
5934 if ((change->kind == svn_fs_path_change_add)
5935 && (old_change->change_kind != svn_fs_path_change_delete)
5936 && (old_change->change_kind != svn_fs_path_change_reset))
5937 return svn_error_create
5938 (SVN_ERR_FS_CORRUPT, NULL,
5939 _("Invalid change ordering: add change on preexisting path"));
5941 /* Now, merge that change in. */
5942 switch (change->kind)
5944 case svn_fs_path_change_reset:
5945 /* A reset here will simply remove the path change from the
5950 case svn_fs_path_change_delete:
5951 if (old_change->change_kind == svn_fs_path_change_add)
5953 /* If the path was introduced in this transaction via an
5954 add, and we are deleting it, just remove the path
5960 /* A deletion overrules all previous changes. */
5961 old_change->change_kind = svn_fs_path_change_delete;
5962 old_change->text_mod = change->text_mod;
5963 old_change->prop_mod = change->prop_mod;
5964 old_change->copyfrom_rev = SVN_INVALID_REVNUM;
5965 old_change->copyfrom_path = NULL;
5969 case svn_fs_path_change_add:
5970 case svn_fs_path_change_replace:
5971 /* An add at this point must be following a previous delete,
5972 so treat it just like a replace. */
5973 old_change->change_kind = svn_fs_path_change_replace;
5974 old_change->node_rev_id = svn_fs_fs__id_copy(change->noderev_id,
5976 old_change->text_mod = change->text_mod;
5977 old_change->prop_mod = change->prop_mod;
5978 if (change->copyfrom_rev == SVN_INVALID_REVNUM)
5980 old_change->copyfrom_rev = SVN_INVALID_REVNUM;
5981 old_change->copyfrom_path = NULL;
5985 old_change->copyfrom_rev = change->copyfrom_rev;
5986 old_change->copyfrom_path = apr_pstrdup(pool,
5987 change->copyfrom_path);
5991 case svn_fs_path_change_modify:
5993 if (change->text_mod)
5994 old_change->text_mod = TRUE;
5995 if (change->prop_mod)
5996 old_change->prop_mod = TRUE;
6000 /* Point our new_change to our (possibly modified) old_change. */
6001 new_change = old_change;
6005 /* This change is new to the hash, so make a new public change
6006 structure from the internal one (in the hash's pool), and dup
6007 the path into the hash's pool, too. */
6008 new_change = apr_pcalloc(pool, sizeof(*new_change));
6009 new_change->node_rev_id = svn_fs_fs__id_copy(change->noderev_id, pool);
6010 new_change->change_kind = change->kind;
6011 new_change->text_mod = change->text_mod;
6012 new_change->prop_mod = change->prop_mod;
6013 /* In FSFS, copyfrom_known is *always* true, since we've always
6014 * stored copyfroms in changed paths lists. */
6015 new_change->copyfrom_known = TRUE;
6016 if (change->copyfrom_rev != SVN_INVALID_REVNUM)
6018 new_change->copyfrom_rev = change->copyfrom_rev;
6019 new_change->copyfrom_path = apr_pstrdup(pool, change->copyfrom_path);
6023 new_change->copyfrom_rev = SVN_INVALID_REVNUM;
6024 new_change->copyfrom_path = NULL;
6029 new_change->node_kind = change->node_kind;
6031 /* Add (or update) this path.
6033 Note: this key might already be present, and it would be nice to
6034 re-use its value, but there is no way to fetch it. The API makes no
6035 guarantees that this (new) key will not be retained. Thus, we (again)
6036 copy the key into the target pool to ensure a proper lifetime. */
6037 path = apr_pstrmemdup(pool, change->path, path_len);
6038 apr_hash_set(changes, path, path_len, new_change);
6040 /* Update the copyfrom cache, if any. */
6043 apr_pool_t *copyfrom_pool = apr_hash_pool_get(copyfrom_cache);
6044 const char *copyfrom_string = NULL, *copyfrom_key = path;
6047 if (SVN_IS_VALID_REVNUM(new_change->copyfrom_rev))
6048 copyfrom_string = apr_psprintf(copyfrom_pool, "%ld %s",
6049 new_change->copyfrom_rev,
6050 new_change->copyfrom_path);
6052 copyfrom_string = "";
6054 /* We need to allocate a copy of the key in the copyfrom_pool if
6055 * we're not doing a deletion and if it isn't already there. */
6056 if ( copyfrom_string
6057 && ( ! apr_hash_count(copyfrom_cache)
6058 || ! apr_hash_get(copyfrom_cache, copyfrom_key, path_len)))
6059 copyfrom_key = apr_pstrmemdup(copyfrom_pool, copyfrom_key, path_len);
6061 apr_hash_set(copyfrom_cache, copyfrom_key, path_len,
6065 return SVN_NO_ERROR;
6068 /* The 256 is an arbitrary size large enough to hold the node id and the
6070 #define MAX_CHANGE_LINE_LEN FSFS_MAX_PATH_LEN + 256
6072 /* Read the next entry in the changes record from file FILE and store
6073 the resulting change in *CHANGE_P. If there is no next record,
6074 store NULL there. Perform all allocations from POOL. */
6075 static svn_error_t *
6076 read_change(change_t **change_p,
6080 char buf[MAX_CHANGE_LINE_LEN];
6081 apr_size_t len = sizeof(buf);
6083 char *str, *last_str = buf, *kind_str;
6086 /* Default return value. */
6089 err = svn_io_read_length_line(file, buf, &len, pool);
6091 /* Check for a blank line. */
6092 if (err || (len == 0))
6094 if (err && APR_STATUS_IS_EOF(err->apr_err))
6096 svn_error_clear(err);
6097 return SVN_NO_ERROR;
6099 if ((len == 0) && (! err))
6100 return SVN_NO_ERROR;
6101 return svn_error_trace(err);
6104 change = apr_pcalloc(pool, sizeof(*change));
6106 /* Get the node-id of the change. */
6107 str = svn_cstring_tokenize(" ", &last_str);
6109 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6110 _("Invalid changes line in rev-file"));
6112 change->noderev_id = svn_fs_fs__id_parse(str, strlen(str), pool);
6113 if (change->noderev_id == NULL)
6114 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6115 _("Invalid changes line in rev-file"));
6117 /* Get the change type. */
6118 str = svn_cstring_tokenize(" ", &last_str);
6120 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6121 _("Invalid changes line in rev-file"));
6123 /* Don't bother to check the format number before looking for
6124 * node-kinds: just read them if you find them. */
6125 change->node_kind = svn_node_unknown;
6126 kind_str = strchr(str, '-');
6129 /* Cap off the end of "str" (the action). */
6132 if (strcmp(kind_str, KIND_FILE) == 0)
6133 change->node_kind = svn_node_file;
6134 else if (strcmp(kind_str, KIND_DIR) == 0)
6135 change->node_kind = svn_node_dir;
6137 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6138 _("Invalid changes line in rev-file"));
6141 if (strcmp(str, ACTION_MODIFY) == 0)
6143 change->kind = svn_fs_path_change_modify;
6145 else if (strcmp(str, ACTION_ADD) == 0)
6147 change->kind = svn_fs_path_change_add;
6149 else if (strcmp(str, ACTION_DELETE) == 0)
6151 change->kind = svn_fs_path_change_delete;
6153 else if (strcmp(str, ACTION_REPLACE) == 0)
6155 change->kind = svn_fs_path_change_replace;
6157 else if (strcmp(str, ACTION_RESET) == 0)
6159 change->kind = svn_fs_path_change_reset;
6163 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6164 _("Invalid change kind in rev file"));
6167 /* Get the text-mod flag. */
6168 str = svn_cstring_tokenize(" ", &last_str);
6170 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6171 _("Invalid changes line in rev-file"));
6173 if (strcmp(str, FLAG_TRUE) == 0)
6175 change->text_mod = TRUE;
6177 else if (strcmp(str, FLAG_FALSE) == 0)
6179 change->text_mod = FALSE;
6183 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6184 _("Invalid text-mod flag in rev-file"));
6187 /* Get the prop-mod flag. */
6188 str = svn_cstring_tokenize(" ", &last_str);
6190 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6191 _("Invalid changes line in rev-file"));
6193 if (strcmp(str, FLAG_TRUE) == 0)
6195 change->prop_mod = TRUE;
6197 else if (strcmp(str, FLAG_FALSE) == 0)
6199 change->prop_mod = FALSE;
6203 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6204 _("Invalid prop-mod flag in rev-file"));
6207 /* Get the changed path. */
6208 change->path = apr_pstrdup(pool, last_str);
6211 /* Read the next line, the copyfrom line. */
6213 SVN_ERR(svn_io_read_length_line(file, buf, &len, pool));
6217 change->copyfrom_rev = SVN_INVALID_REVNUM;
6218 change->copyfrom_path = NULL;
6223 str = svn_cstring_tokenize(" ", &last_str);
6225 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6226 _("Invalid changes line in rev-file"));
6227 change->copyfrom_rev = SVN_STR_TO_REV(str);
6230 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6231 _("Invalid changes line in rev-file"));
6233 change->copyfrom_path = apr_pstrdup(pool, last_str);
6238 return SVN_NO_ERROR;
6241 /* Examine all the changed path entries in CHANGES and store them in
6242 *CHANGED_PATHS. Folding is done to remove redundant or unnecessary
6243 *data. Store a hash of paths to copyfrom "REV PATH" strings in
6244 COPYFROM_HASH if it is non-NULL. If PREFOLDED is true, assume that
6245 the changed-path entries have already been folded (by
6246 write_final_changed_path_info) and may be out of order, so we shouldn't
6247 remove children of replaced or deleted directories. Do all
6248 allocations in POOL. */
6249 static svn_error_t *
6250 process_changes(apr_hash_t *changed_paths,
6251 apr_hash_t *copyfrom_cache,
6252 apr_array_header_t *changes,
6253 svn_boolean_t prefolded,
6256 apr_pool_t *iterpool = svn_pool_create(pool);
6259 /* Read in the changes one by one, folding them into our local hash
6262 for (i = 0; i < changes->nelts; ++i)
6264 change_t *change = APR_ARRAY_IDX(changes, i, change_t *);
6266 SVN_ERR(fold_change(changed_paths, change, copyfrom_cache));
6268 /* Now, if our change was a deletion or replacement, we have to
6269 blow away any changes thus far on paths that are (or, were)
6270 children of this path.
6271 ### i won't bother with another iteration pool here -- at
6272 most we talking about a few extra dups of paths into what
6273 is already a temporary subpool.
6276 if (((change->kind == svn_fs_path_change_delete)
6277 || (change->kind == svn_fs_path_change_replace))
6280 apr_hash_index_t *hi;
6282 /* a potential child path must contain at least 2 more chars
6283 (the path separator plus at least one char for the name).
6284 Also, we should not assume that all paths have been normalized
6285 i.e. some might have trailing path separators.
6287 apr_ssize_t change_path_len = strlen(change->path);
6288 apr_ssize_t min_child_len = change_path_len == 0
6290 : change->path[change_path_len-1] == '/'
6291 ? change_path_len + 1
6292 : change_path_len + 2;
6294 /* CAUTION: This is the inner loop of an O(n^2) algorithm.
6295 The number of changes to process may be >> 1000.
6296 Therefore, keep the inner loop as tight as possible.
6298 for (hi = apr_hash_first(iterpool, changed_paths);
6300 hi = apr_hash_next(hi))
6302 /* KEY is the path. */
6305 apr_hash_this(hi, &path, &klen, NULL);
6307 /* If we come across a child of our path, remove it.
6308 Call svn_dirent_is_child only if there is a chance that
6309 this is actually a sub-path.
6311 if ( klen >= min_child_len
6312 && svn_dirent_is_child(change->path, path, iterpool))
6313 apr_hash_set(changed_paths, path, klen, NULL);
6317 /* Clear the per-iteration subpool. */
6318 svn_pool_clear(iterpool);
6321 /* Destroy the per-iteration subpool. */
6322 svn_pool_destroy(iterpool);
6324 return SVN_NO_ERROR;
6327 /* Fetch all the changes from FILE and store them in *CHANGES. Do all
6328 allocations in POOL. */
6329 static svn_error_t *
6330 read_all_changes(apr_array_header_t **changes,
6336 /* pre-allocate enough room for most change lists
6337 (will be auto-expanded as necessary) */
6338 *changes = apr_array_make(pool, 30, sizeof(change_t *));
6340 SVN_ERR(read_change(&change, file, pool));
6343 APR_ARRAY_PUSH(*changes, change_t*) = change;
6344 SVN_ERR(read_change(&change, file, pool));
6347 return SVN_NO_ERROR;
6351 svn_fs_fs__txn_changes_fetch(apr_hash_t **changed_paths_p,
6357 apr_hash_t *changed_paths = apr_hash_make(pool);
6358 apr_array_header_t *changes;
6359 apr_pool_t *scratch_pool = svn_pool_create(pool);
6361 SVN_ERR(svn_io_file_open(&file, path_txn_changes(fs, txn_id, pool),
6362 APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
6364 SVN_ERR(read_all_changes(&changes, file, scratch_pool));
6365 SVN_ERR(process_changes(changed_paths, NULL, changes, FALSE, pool));
6366 svn_pool_destroy(scratch_pool);
6368 SVN_ERR(svn_io_file_close(file, pool));
6370 *changed_paths_p = changed_paths;
6372 return SVN_NO_ERROR;
6375 /* Fetch the list of change in revision REV in FS and return it in *CHANGES.
6376 * Allocate the result in POOL.
6378 static svn_error_t *
6379 get_changes(apr_array_header_t **changes,
6384 apr_off_t changes_offset;
6385 apr_file_t *revision_file;
6386 svn_boolean_t found;
6387 fs_fs_data_t *ffd = fs->fsap_data;
6389 /* try cache lookup first */
6391 if (ffd->changes_cache)
6393 SVN_ERR(svn_cache__get((void **) changes, &found, ffd->changes_cache,
6396 return SVN_NO_ERROR;
6399 /* read changes from revision file */
6401 SVN_ERR(ensure_revision_exists(fs, rev, pool));
6403 SVN_ERR(open_pack_or_rev_file(&revision_file, fs, rev, pool));
6405 SVN_ERR(get_root_changes_offset(NULL, &changes_offset, revision_file, fs,
6408 SVN_ERR(svn_io_file_seek(revision_file, APR_SET, &changes_offset, pool));
6409 SVN_ERR(read_all_changes(changes, revision_file, pool));
6411 SVN_ERR(svn_io_file_close(revision_file, pool));
6413 /* cache for future reference */
6415 if (ffd->changes_cache)
6416 SVN_ERR(svn_cache__set(ffd->changes_cache, &rev, *changes, pool));
6418 return SVN_NO_ERROR;
6423 svn_fs_fs__paths_changed(apr_hash_t **changed_paths_p,
6426 apr_hash_t *copyfrom_cache,
6429 apr_hash_t *changed_paths;
6430 apr_array_header_t *changes;
6431 apr_pool_t *scratch_pool = svn_pool_create(pool);
6433 SVN_ERR(get_changes(&changes, fs, rev, scratch_pool));
6435 changed_paths = svn_hash__make(pool);
6437 SVN_ERR(process_changes(changed_paths, copyfrom_cache, changes,
6439 svn_pool_destroy(scratch_pool);
6441 *changed_paths_p = changed_paths;
6443 return SVN_NO_ERROR;
6446 /* Copy a revision node-rev SRC into the current transaction TXN_ID in
6447 the filesystem FS. This is only used to create the root of a transaction.
6448 Allocations are from POOL. */
6449 static svn_error_t *
6450 create_new_txn_noderev_from_rev(svn_fs_t *fs,
6455 node_revision_t *noderev;
6456 const char *node_id, *copy_id;
6458 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, src, pool));
6460 if (svn_fs_fs__id_txn_id(noderev->id))
6461 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6462 _("Copying from transactions not allowed"));
6464 noderev->predecessor_id = noderev->id;
6465 noderev->predecessor_count++;
6466 noderev->copyfrom_path = NULL;
6467 noderev->copyfrom_rev = SVN_INVALID_REVNUM;
6469 /* For the transaction root, the copyroot never changes. */
6471 node_id = svn_fs_fs__id_node_id(noderev->id);
6472 copy_id = svn_fs_fs__id_copy_id(noderev->id);
6473 noderev->id = svn_fs_fs__id_txn_create(node_id, copy_id, txn_id, pool);
6475 return svn_fs_fs__put_node_revision(fs, noderev->id, noderev, TRUE, pool);
6478 /* A structure used by get_and_increment_txn_key_body(). */
6479 struct get_and_increment_txn_key_baton {
6485 /* Callback used in the implementation of create_txn_dir(). This gets
6486 the current base 36 value in PATH_TXN_CURRENT and increments it.
6487 It returns the original value by the baton. */
6488 static svn_error_t *
6489 get_and_increment_txn_key_body(void *baton, apr_pool_t *pool)
6491 struct get_and_increment_txn_key_baton *cb = baton;
6492 const char *txn_current_filename = path_txn_current(cb->fs, pool);
6493 const char *tmp_filename;
6494 char next_txn_id[MAX_KEY_SIZE+3];
6497 svn_stringbuf_t *buf;
6498 SVN_ERR(read_content(&buf, txn_current_filename, cb->pool));
6500 /* remove trailing newlines */
6501 svn_stringbuf_strip_whitespace(buf);
6502 cb->txn_id = buf->data;
6505 /* Increment the key and add a trailing \n to the string so the
6506 txn-current file has a newline in it. */
6507 svn_fs_fs__next_key(cb->txn_id, &len, next_txn_id);
6508 next_txn_id[len] = '\n';
6510 next_txn_id[len] = '\0';
6512 SVN_ERR(svn_io_write_unique(&tmp_filename,
6513 svn_dirent_dirname(txn_current_filename, pool),
6514 next_txn_id, len, svn_io_file_del_none, pool));
6515 SVN_ERR(move_into_place(tmp_filename, txn_current_filename,
6516 txn_current_filename, pool));
6518 return SVN_NO_ERROR;
6521 /* Create a unique directory for a transaction in FS based on revision
6522 REV. Return the ID for this transaction in *ID_P. Use a sequence
6523 value in the transaction ID to prevent reuse of transaction IDs. */
6524 static svn_error_t *
6525 create_txn_dir(const char **id_p, svn_fs_t *fs, svn_revnum_t rev,
6528 struct get_and_increment_txn_key_baton cb;
6529 const char *txn_dir;
6531 /* Get the current transaction sequence value, which is a base-36
6532 number, from the txn-current file, and write an
6533 incremented value back out to the file. Place the revision
6534 number the transaction is based off into the transaction id. */
6537 SVN_ERR(with_txn_current_lock(fs,
6538 get_and_increment_txn_key_body,
6541 *id_p = apr_psprintf(pool, "%ld-%s", rev, cb.txn_id);
6543 txn_dir = svn_dirent_join_many(pool,
6546 apr_pstrcat(pool, *id_p, PATH_EXT_TXN,
6550 return svn_io_dir_make(txn_dir, APR_OS_DEFAULT, pool);
6553 /* Create a unique directory for a transaction in FS based on revision
6554 REV. Return the ID for this transaction in *ID_P. This
6555 implementation is used in svn 1.4 and earlier repositories and is
6556 kept in 1.5 and greater to support the --pre-1.4-compatible and
6557 --pre-1.5-compatible repository creation options. Reused
6558 transaction IDs are possible with this implementation. */
6559 static svn_error_t *
6560 create_txn_dir_pre_1_5(const char **id_p, svn_fs_t *fs, svn_revnum_t rev,
6564 apr_pool_t *subpool;
6565 const char *unique_path, *prefix;
6567 /* Try to create directories named "<txndir>/<rev>-<uniqueifier>.txn". */
6568 prefix = svn_dirent_join_many(pool, fs->path, PATH_TXNS_DIR,
6569 apr_psprintf(pool, "%ld", rev), NULL);
6571 subpool = svn_pool_create(pool);
6572 for (i = 1; i <= 99999; i++)
6576 svn_pool_clear(subpool);
6577 unique_path = apr_psprintf(subpool, "%s-%u" PATH_EXT_TXN, prefix, i);
6578 err = svn_io_dir_make(unique_path, APR_OS_DEFAULT, subpool);
6581 /* We succeeded. Return the basename minus the ".txn" extension. */
6582 const char *name = svn_dirent_basename(unique_path, subpool);
6583 *id_p = apr_pstrndup(pool, name,
6584 strlen(name) - strlen(PATH_EXT_TXN));
6585 svn_pool_destroy(subpool);
6586 return SVN_NO_ERROR;
6588 if (! APR_STATUS_IS_EEXIST(err->apr_err))
6589 return svn_error_trace(err);
6590 svn_error_clear(err);
6593 return svn_error_createf(SVN_ERR_IO_UNIQUE_NAMES_EXHAUSTED,
6595 _("Unable to create transaction directory "
6596 "in '%s' for revision %ld"),
6597 svn_dirent_local_style(fs->path, pool),
6602 svn_fs_fs__create_txn(svn_fs_txn_t **txn_p,
6607 fs_fs_data_t *ffd = fs->fsap_data;
6609 svn_fs_id_t *root_id;
6611 txn = apr_pcalloc(pool, sizeof(*txn));
6613 /* Get the txn_id. */
6614 if (ffd->format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT)
6615 SVN_ERR(create_txn_dir(&txn->id, fs, rev, pool));
6617 SVN_ERR(create_txn_dir_pre_1_5(&txn->id, fs, rev, pool));
6620 txn->base_rev = rev;
6622 txn->vtable = &txn_vtable;
6625 /* Create a new root node for this transaction. */
6626 SVN_ERR(svn_fs_fs__rev_get_root(&root_id, fs, rev, pool));
6627 SVN_ERR(create_new_txn_noderev_from_rev(fs, txn->id, root_id, pool));
6629 /* Create an empty rev file. */
6630 SVN_ERR(svn_io_file_create(path_txn_proto_rev(fs, txn->id, pool), "",
6633 /* Create an empty rev-lock file. */
6634 SVN_ERR(svn_io_file_create(path_txn_proto_rev_lock(fs, txn->id, pool), "",
6637 /* Create an empty changes file. */
6638 SVN_ERR(svn_io_file_create(path_txn_changes(fs, txn->id, pool), "",
6641 /* Create the next-ids file. */
6642 return svn_io_file_create(path_txn_next_ids(fs, txn->id, pool), "0 0\n",
6646 /* Store the property list for transaction TXN_ID in PROPLIST.
6647 Perform temporary allocations in POOL. */
6648 static svn_error_t *
6649 get_txn_proplist(apr_hash_t *proplist,
6654 svn_stream_t *stream;
6656 /* Check for issue #3696. (When we find and fix the cause, we can change
6657 * this to an assertion.) */
6659 return svn_error_create(SVN_ERR_INCORRECT_PARAMS, NULL,
6660 _("Internal error: a null transaction id was "
6661 "passed to get_txn_proplist()"));
6663 /* Open the transaction properties file. */
6664 SVN_ERR(svn_stream_open_readonly(&stream, path_txn_props(fs, txn_id, pool),
6667 /* Read in the property list. */
6668 SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool));
6670 return svn_stream_close(stream);
6674 svn_fs_fs__change_txn_prop(svn_fs_txn_t *txn,
6676 const svn_string_t *value,
6679 apr_array_header_t *props = apr_array_make(pool, 1, sizeof(svn_prop_t));
6684 APR_ARRAY_PUSH(props, svn_prop_t) = prop;
6686 return svn_fs_fs__change_txn_props(txn, props, pool);
6690 svn_fs_fs__change_txn_props(svn_fs_txn_t *txn,
6691 const apr_array_header_t *props,
6694 const char *txn_prop_filename;
6695 svn_stringbuf_t *buf;
6696 svn_stream_t *stream;
6697 apr_hash_t *txn_prop = apr_hash_make(pool);
6701 err = get_txn_proplist(txn_prop, txn->fs, txn->id, pool);
6702 /* Here - and here only - we need to deal with the possibility that the
6703 transaction property file doesn't yet exist. The rest of the
6704 implementation assumes that the file exists, but we're called to set the
6705 initial transaction properties as the transaction is being created. */
6706 if (err && (APR_STATUS_IS_ENOENT(err->apr_err)))
6707 svn_error_clear(err);
6709 return svn_error_trace(err);
6711 for (i = 0; i < props->nelts; i++)
6713 svn_prop_t *prop = &APR_ARRAY_IDX(props, i, svn_prop_t);
6715 svn_hash_sets(txn_prop, prop->name, prop->value);
6718 /* Create a new version of the file and write out the new props. */
6719 /* Open the transaction properties file. */
6720 buf = svn_stringbuf_create_ensure(1024, pool);
6721 stream = svn_stream_from_stringbuf(buf, pool);
6722 SVN_ERR(svn_hash_write2(txn_prop, stream, SVN_HASH_TERMINATOR, pool));
6723 SVN_ERR(svn_stream_close(stream));
6724 SVN_ERR(svn_io_write_unique(&txn_prop_filename,
6725 path_txn_dir(txn->fs, txn->id, pool),
6728 svn_io_file_del_none,
6730 return svn_io_file_rename(txn_prop_filename,
6731 path_txn_props(txn->fs, txn->id, pool),
6736 svn_fs_fs__get_txn(transaction_t **txn_p,
6742 node_revision_t *noderev;
6743 svn_fs_id_t *root_id;
6745 txn = apr_pcalloc(pool, sizeof(*txn));
6746 txn->proplist = apr_hash_make(pool);
6748 SVN_ERR(get_txn_proplist(txn->proplist, fs, txn_id, pool));
6749 root_id = svn_fs_fs__id_txn_create("0", "0", txn_id, pool);
6751 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, root_id, pool));
6753 txn->root_id = svn_fs_fs__id_copy(noderev->id, pool);
6754 txn->base_id = svn_fs_fs__id_copy(noderev->predecessor_id, pool);
6759 return SVN_NO_ERROR;
6762 /* Write out the currently available next node_id NODE_ID and copy_id
6763 COPY_ID for transaction TXN_ID in filesystem FS. The next node-id is
6764 used both for creating new unique nodes for the given transaction, as
6765 well as uniquifying representations. Perform temporary allocations in
6767 static svn_error_t *
6768 write_next_ids(svn_fs_t *fs,
6770 const char *node_id,
6771 const char *copy_id,
6775 svn_stream_t *out_stream;
6777 SVN_ERR(svn_io_file_open(&file, path_txn_next_ids(fs, txn_id, pool),
6778 APR_WRITE | APR_TRUNCATE,
6779 APR_OS_DEFAULT, pool));
6781 out_stream = svn_stream_from_aprfile2(file, TRUE, pool);
6783 SVN_ERR(svn_stream_printf(out_stream, pool, "%s %s\n", node_id, copy_id));
6785 SVN_ERR(svn_stream_close(out_stream));
6786 return svn_io_file_close(file, pool);
6789 /* Find out what the next unique node-id and copy-id are for
6790 transaction TXN_ID in filesystem FS. Store the results in *NODE_ID
6791 and *COPY_ID. The next node-id is used both for creating new unique
6792 nodes for the given transaction, as well as uniquifying representations.
6793 Perform all allocations in POOL. */
6794 static svn_error_t *
6795 read_next_ids(const char **node_id,
6796 const char **copy_id,
6802 char buf[MAX_KEY_SIZE*2+3];
6804 char *str, *last_str = buf;
6806 SVN_ERR(svn_io_file_open(&file, path_txn_next_ids(fs, txn_id, pool),
6807 APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool));
6809 limit = sizeof(buf);
6810 SVN_ERR(svn_io_read_length_line(file, buf, &limit, pool));
6812 SVN_ERR(svn_io_file_close(file, pool));
6814 /* Parse this into two separate strings. */
6816 str = svn_cstring_tokenize(" ", &last_str);
6818 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6819 _("next-id file corrupt"));
6821 *node_id = apr_pstrdup(pool, str);
6823 str = svn_cstring_tokenize(" ", &last_str);
6825 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
6826 _("next-id file corrupt"));
6828 *copy_id = apr_pstrdup(pool, str);
6830 return SVN_NO_ERROR;
6833 /* Get a new and unique to this transaction node-id for transaction
6834 TXN_ID in filesystem FS. Store the new node-id in *NODE_ID_P.
6835 Node-ids are guaranteed to be unique to this transction, but may
6836 not necessarily be sequential. Perform all allocations in POOL. */
6837 static svn_error_t *
6838 get_new_txn_node_id(const char **node_id_p,
6843 const char *cur_node_id, *cur_copy_id;
6847 /* First read in the current next-ids file. */
6848 SVN_ERR(read_next_ids(&cur_node_id, &cur_copy_id, fs, txn_id, pool));
6850 node_id = apr_pcalloc(pool, strlen(cur_node_id) + 2);
6852 len = strlen(cur_node_id);
6853 svn_fs_fs__next_key(cur_node_id, &len, node_id);
6855 SVN_ERR(write_next_ids(fs, txn_id, node_id, cur_copy_id, pool));
6857 *node_id_p = apr_pstrcat(pool, "_", cur_node_id, (char *)NULL);
6859 return SVN_NO_ERROR;
6863 svn_fs_fs__create_node(const svn_fs_id_t **id_p,
6865 node_revision_t *noderev,
6866 const char *copy_id,
6870 const char *node_id;
6871 const svn_fs_id_t *id;
6873 /* Get a new node-id for this node. */
6874 SVN_ERR(get_new_txn_node_id(&node_id, fs, txn_id, pool));
6876 id = svn_fs_fs__id_txn_create(node_id, copy_id, txn_id, pool);
6880 SVN_ERR(svn_fs_fs__put_node_revision(fs, noderev->id, noderev, FALSE, pool));
6884 return SVN_NO_ERROR;
6888 svn_fs_fs__purge_txn(svn_fs_t *fs,
6892 fs_fs_data_t *ffd = fs->fsap_data;
6894 /* Remove the shared transaction object associated with this transaction. */
6895 SVN_ERR(purge_shared_txn(fs, txn_id, pool));
6896 /* Remove the directory associated with this transaction. */
6897 SVN_ERR(svn_io_remove_dir2(path_txn_dir(fs, txn_id, pool), FALSE,
6899 if (ffd->format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT)
6901 /* Delete protorev and its lock, which aren't in the txn
6902 directory. It's OK if they don't exist (for example, if this
6903 is post-commit and the proto-rev has been moved into
6905 SVN_ERR(svn_io_remove_file2(path_txn_proto_rev(fs, txn_id, pool),
6907 SVN_ERR(svn_io_remove_file2(path_txn_proto_rev_lock(fs, txn_id, pool),
6910 return SVN_NO_ERROR;
6915 svn_fs_fs__abort_txn(svn_fs_txn_t *txn,
6918 SVN_ERR(svn_fs__check_fs(txn->fs, TRUE));
6920 /* Now, purge the transaction. */
6921 SVN_ERR_W(svn_fs_fs__purge_txn(txn->fs, txn->id, pool),
6922 apr_psprintf(pool, _("Transaction '%s' cleanup failed"),
6925 return SVN_NO_ERROR;
6930 svn_fs_fs__set_entry(svn_fs_t *fs,
6932 node_revision_t *parent_noderev,
6934 const svn_fs_id_t *id,
6935 svn_node_kind_t kind,
6938 representation_t *rep = parent_noderev->data_rep;
6939 const char *filename = path_txn_node_children(fs, parent_noderev->id, pool);
6942 fs_fs_data_t *ffd = fs->fsap_data;
6943 apr_pool_t *subpool = svn_pool_create(pool);
6945 if (!rep || !rep->txn_id)
6947 const char *unique_suffix;
6948 apr_hash_t *entries;
6950 /* Before we can modify the directory, we need to dump its old
6951 contents into a mutable representation file. */
6952 SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, parent_noderev,
6954 SVN_ERR(unparse_dir_entries(&entries, entries, subpool));
6955 SVN_ERR(svn_io_file_open(&file, filename,
6956 APR_WRITE | APR_CREATE | APR_BUFFERED,
6957 APR_OS_DEFAULT, pool));
6958 out = svn_stream_from_aprfile2(file, TRUE, pool);
6959 SVN_ERR(svn_hash_write2(entries, out, SVN_HASH_TERMINATOR, subpool));
6961 svn_pool_clear(subpool);
6963 /* Mark the node-rev's data rep as mutable. */
6964 rep = apr_pcalloc(pool, sizeof(*rep));
6965 rep->revision = SVN_INVALID_REVNUM;
6966 rep->txn_id = txn_id;
6967 SVN_ERR(get_new_txn_node_id(&unique_suffix, fs, txn_id, pool));
6968 rep->uniquifier = apr_psprintf(pool, "%s/%s", txn_id, unique_suffix);
6969 parent_noderev->data_rep = rep;
6970 SVN_ERR(svn_fs_fs__put_node_revision(fs, parent_noderev->id,
6971 parent_noderev, FALSE, pool));
6975 /* The directory rep is already mutable, so just open it for append. */
6976 SVN_ERR(svn_io_file_open(&file, filename, APR_WRITE | APR_APPEND,
6977 APR_OS_DEFAULT, pool));
6978 out = svn_stream_from_aprfile2(file, TRUE, pool);
6981 /* if we have a directory cache for this transaction, update it */
6982 if (ffd->txn_dir_cache)
6984 /* build parameters: (name, new entry) pair */
6986 svn_fs_fs__id_unparse(parent_noderev->id, subpool)->data;
6987 replace_baton_t baton;
6990 baton.new_entry = NULL;
6994 baton.new_entry = apr_pcalloc(subpool, sizeof(*baton.new_entry));
6995 baton.new_entry->name = name;
6996 baton.new_entry->kind = kind;
6997 baton.new_entry->id = id;
7000 /* actually update the cached directory (if cached) */
7001 SVN_ERR(svn_cache__set_partial(ffd->txn_dir_cache, key,
7002 svn_fs_fs__replace_dir_entry, &baton,
7005 svn_pool_clear(subpool);
7007 /* Append an incremental hash entry for the entry change. */
7010 const char *val = unparse_dir_entry(kind, id, subpool);
7012 SVN_ERR(svn_stream_printf(out, subpool, "K %" APR_SIZE_T_FMT "\n%s\n"
7013 "V %" APR_SIZE_T_FMT "\n%s\n",
7019 SVN_ERR(svn_stream_printf(out, subpool, "D %" APR_SIZE_T_FMT "\n%s\n",
7020 strlen(name), name));
7023 SVN_ERR(svn_io_file_close(file, subpool));
7024 svn_pool_destroy(subpool);
7025 return SVN_NO_ERROR;
7028 /* Write a single change entry, path PATH, change CHANGE, and copyfrom
7029 string COPYFROM, into the file specified by FILE. Only include the
7030 node kind field if INCLUDE_NODE_KIND is true. All temporary
7031 allocations are in POOL. */
7032 static svn_error_t *
7033 write_change_entry(apr_file_t *file,
7035 svn_fs_path_change2_t *change,
7036 svn_boolean_t include_node_kind,
7039 const char *idstr, *buf;
7040 const char *change_string = NULL;
7041 const char *kind_string = "";
7043 switch (change->change_kind)
7045 case svn_fs_path_change_modify:
7046 change_string = ACTION_MODIFY;
7048 case svn_fs_path_change_add:
7049 change_string = ACTION_ADD;
7051 case svn_fs_path_change_delete:
7052 change_string = ACTION_DELETE;
7054 case svn_fs_path_change_replace:
7055 change_string = ACTION_REPLACE;
7057 case svn_fs_path_change_reset:
7058 change_string = ACTION_RESET;
7061 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
7062 _("Invalid change type %d"),
7063 change->change_kind);
7066 if (change->node_rev_id)
7067 idstr = svn_fs_fs__id_unparse(change->node_rev_id, pool)->data;
7069 idstr = ACTION_RESET;
7071 if (include_node_kind)
7073 SVN_ERR_ASSERT(change->node_kind == svn_node_dir
7074 || change->node_kind == svn_node_file);
7075 kind_string = apr_psprintf(pool, "-%s",
7076 change->node_kind == svn_node_dir
7077 ? KIND_DIR : KIND_FILE);
7079 buf = apr_psprintf(pool, "%s %s%s %s %s %s\n",
7080 idstr, change_string, kind_string,
7081 change->text_mod ? FLAG_TRUE : FLAG_FALSE,
7082 change->prop_mod ? FLAG_TRUE : FLAG_FALSE,
7085 SVN_ERR(svn_io_file_write_full(file, buf, strlen(buf), NULL, pool));
7087 if (SVN_IS_VALID_REVNUM(change->copyfrom_rev))
7089 buf = apr_psprintf(pool, "%ld %s", change->copyfrom_rev,
7090 change->copyfrom_path);
7091 SVN_ERR(svn_io_file_write_full(file, buf, strlen(buf), NULL, pool));
7094 return svn_io_file_write_full(file, "\n", 1, NULL, pool);
7098 svn_fs_fs__add_change(svn_fs_t *fs,
7101 const svn_fs_id_t *id,
7102 svn_fs_path_change_kind_t change_kind,
7103 svn_boolean_t text_mod,
7104 svn_boolean_t prop_mod,
7105 svn_node_kind_t node_kind,
7106 svn_revnum_t copyfrom_rev,
7107 const char *copyfrom_path,
7111 svn_fs_path_change2_t *change;
7113 SVN_ERR(svn_io_file_open(&file, path_txn_changes(fs, txn_id, pool),
7114 APR_APPEND | APR_WRITE | APR_CREATE
7115 | APR_BUFFERED, APR_OS_DEFAULT, pool));
7117 change = svn_fs__path_change_create_internal(id, change_kind, pool);
7118 change->text_mod = text_mod;
7119 change->prop_mod = prop_mod;
7120 change->node_kind = node_kind;
7121 change->copyfrom_rev = copyfrom_rev;
7122 change->copyfrom_path = apr_pstrdup(pool, copyfrom_path);
7124 SVN_ERR(write_change_entry(file, path, change, TRUE, pool));
7126 return svn_io_file_close(file, pool);
7129 /* This baton is used by the representation writing streams. It keeps
7130 track of the checksum information as well as the total size of the
7131 representation so far. */
7132 struct rep_write_baton
7134 /* The FS we are writing to. */
7137 /* Actual file to which we are writing. */
7138 svn_stream_t *rep_stream;
7140 /* A stream from the delta combiner. Data written here gets
7141 deltified, then eventually written to rep_stream. */
7142 svn_stream_t *delta_stream;
7144 /* Where is this representation header stored. */
7145 apr_off_t rep_offset;
7147 /* Start of the actual data. */
7148 apr_off_t delta_start;
7150 /* How many bytes have been written to this rep already. */
7151 svn_filesize_t rep_size;
7153 /* The node revision for which we're writing out info. */
7154 node_revision_t *noderev;
7156 /* Actual output file. */
7158 /* Lock 'cookie' used to unlock the output file once we've finished
7162 svn_checksum_ctx_t *md5_checksum_ctx;
7163 svn_checksum_ctx_t *sha1_checksum_ctx;
7167 apr_pool_t *parent_pool;
7170 /* Handler for the write method of the representation writable stream.
7171 BATON is a rep_write_baton, DATA is the data to write, and *LEN is
7172 the length of this data. */
7173 static svn_error_t *
7174 rep_write_contents(void *baton,
7178 struct rep_write_baton *b = baton;
7180 SVN_ERR(svn_checksum_update(b->md5_checksum_ctx, data, *len));
7181 SVN_ERR(svn_checksum_update(b->sha1_checksum_ctx, data, *len));
7182 b->rep_size += *len;
7184 /* If we are writing a delta, use that stream. */
7185 if (b->delta_stream)
7186 return svn_stream_write(b->delta_stream, data, len);
7188 return svn_stream_write(b->rep_stream, data, len);
7191 /* Given a node-revision NODEREV in filesystem FS, return the
7192 representation in *REP to use as the base for a text representation
7193 delta if PROPS is FALSE. If PROPS has been set, a suitable props
7194 base representation will be returned. Perform temporary allocations
7196 static svn_error_t *
7197 choose_delta_base(representation_t **rep,
7199 node_revision_t *noderev,
7200 svn_boolean_t props,
7205 node_revision_t *base;
7206 fs_fs_data_t *ffd = fs->fsap_data;
7207 svn_boolean_t maybe_shared_rep = FALSE;
7209 /* If we have no predecessors, then use the empty stream as a
7211 if (! noderev->predecessor_count)
7214 return SVN_NO_ERROR;
7217 /* Flip the rightmost '1' bit of the predecessor count to determine
7218 which file rev (counting from 0) we want to use. (To see why
7219 count & (count - 1) unsets the rightmost set bit, think about how
7220 you decrement a binary number.) */
7221 count = noderev->predecessor_count;
7222 count = count & (count - 1);
7224 /* We use skip delta for limiting the number of delta operations
7225 along very long node histories. Close to HEAD however, we create
7226 a linear history to minimize delta size. */
7227 walk = noderev->predecessor_count - count;
7228 if (walk < (int)ffd->max_linear_deltification)
7229 count = noderev->predecessor_count - 1;
7231 /* Finding the delta base over a very long distance can become extremely
7232 expensive for very deep histories, possibly causing client timeouts etc.
7233 OTOH, this is a rare operation and its gains are minimal. Lets simply
7234 start deltification anew close every other 1000 changes or so. */
7235 if (walk > (int)ffd->max_deltification_walk)
7238 return SVN_NO_ERROR;
7241 /* Walk back a number of predecessors equal to the difference
7242 between count and the original predecessor count. (For example,
7243 if noderev has ten predecessors and we want the eighth file rev,
7244 walk back two predecessors.) */
7246 while ((count++) < noderev->predecessor_count)
7248 SVN_ERR(svn_fs_fs__get_node_revision(&base, fs,
7249 base->predecessor_id, pool));
7251 /* If there is a shared rep along the way, we need to limit the
7252 * length of the deltification chain.
7254 * Please note that copied nodes - such as branch directories - will
7255 * look the same (false positive) while reps shared within the same
7256 * revision will not be caught (false negative).
7261 && svn_fs_fs__id_rev(base->id) > base->prop_rep->revision)
7262 maybe_shared_rep = TRUE;
7267 && svn_fs_fs__id_rev(base->id) > base->data_rep->revision)
7268 maybe_shared_rep = TRUE;
7272 /* return a suitable base representation */
7273 *rep = props ? base->prop_rep : base->data_rep;
7275 /* if we encountered a shared rep, it's parent chain may be different
7276 * from the node-rev parent chain. */
7277 if (*rep && maybe_shared_rep)
7279 /* Check whether the length of the deltification chain is acceptable.
7280 * Otherwise, shared reps may form a non-skipping delta chain in
7282 apr_pool_t *sub_pool = svn_pool_create(pool);
7283 representation_t base_rep = **rep;
7285 /* Some reasonable limit, depending on how acceptable longer linear
7286 * chains are in this repo. Also, allow for some minimal chain. */
7287 int max_chain_length = 2 * (int)ffd->max_linear_deltification + 2;
7289 /* re-use open files between iterations */
7290 svn_revnum_t rev_hint = SVN_INVALID_REVNUM;
7291 apr_file_t *file_hint = NULL;
7293 /* follow the delta chain towards the end but for at most
7294 * MAX_CHAIN_LENGTH steps. */
7295 for (; max_chain_length; --max_chain_length)
7297 struct rep_state *rep_state;
7298 struct rep_args *rep_args;
7300 SVN_ERR(create_rep_state_body(&rep_state,
7307 if (!rep_args->is_delta || !rep_args->base_revision)
7310 base_rep.revision = rep_args->base_revision;
7311 base_rep.offset = rep_args->base_offset;
7312 base_rep.size = rep_args->base_length;
7313 base_rep.txn_id = NULL;
7316 /* start new delta chain if the current one has grown too long */
7317 if (max_chain_length == 0)
7320 svn_pool_destroy(sub_pool);
7323 /* verify that the reps don't form a degenerated '*/
7324 return SVN_NO_ERROR;
7327 /* Something went wrong and the pool for the rep write is being
7328 cleared before we've finished writing the rep. So we need
7329 to remove the rep from the protorevfile and we need to unlock
7330 the protorevfile. */
7332 rep_write_cleanup(void *data)
7334 struct rep_write_baton *b = data;
7335 const char *txn_id = svn_fs_fs__id_txn_id(b->noderev->id);
7338 /* Truncate and close the protorevfile. */
7339 err = svn_io_file_trunc(b->file, b->rep_offset, b->pool);
7340 err = svn_error_compose_create(err, svn_io_file_close(b->file, b->pool));
7342 /* Remove our lock regardless of any preceeding errors so that the
7343 being_written flag is always removed and stays consistent with the
7344 file lock which will be removed no matter what since the pool is
7346 err = svn_error_compose_create(err, unlock_proto_rev(b->fs, txn_id,
7347 b->lockcookie, b->pool));
7350 apr_status_t rc = err->apr_err;
7351 svn_error_clear(err);
7359 /* Get a rep_write_baton and store it in *WB_P for the representation
7360 indicated by NODEREV in filesystem FS. Perform allocations in
7361 POOL. Only appropriate for file contents, not for props or
7362 directory contents. */
7363 static svn_error_t *
7364 rep_write_get_baton(struct rep_write_baton **wb_p,
7366 node_revision_t *noderev,
7369 struct rep_write_baton *b;
7371 representation_t *base_rep;
7372 svn_stream_t *source;
7374 svn_txdelta_window_handler_t wh;
7376 fs_fs_data_t *ffd = fs->fsap_data;
7377 int diff_version = ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT ? 1 : 0;
7379 b = apr_pcalloc(pool, sizeof(*b));
7381 b->sha1_checksum_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool);
7382 b->md5_checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool);
7385 b->parent_pool = pool;
7386 b->pool = svn_pool_create(pool);
7388 b->noderev = noderev;
7390 /* Open the prototype rev file and seek to its end. */
7391 SVN_ERR(get_writable_proto_rev(&file, &b->lockcookie,
7392 fs, svn_fs_fs__id_txn_id(noderev->id),
7396 b->rep_stream = svn_stream_from_aprfile2(file, TRUE, b->pool);
7398 SVN_ERR(get_file_offset(&b->rep_offset, file, b->pool));
7400 /* Get the base for this delta. */
7401 SVN_ERR(choose_delta_base(&base_rep, fs, noderev, FALSE, b->pool));
7402 SVN_ERR(read_representation(&source, fs, base_rep, b->pool));
7404 /* Write out the rep header. */
7407 header = apr_psprintf(b->pool, REP_DELTA " %ld %" APR_OFF_T_FMT " %"
7408 SVN_FILESIZE_T_FMT "\n",
7409 base_rep->revision, base_rep->offset,
7414 header = REP_DELTA "\n";
7416 SVN_ERR(svn_io_file_write_full(file, header, strlen(header), NULL,
7419 /* Now determine the offset of the actual svndiff data. */
7420 SVN_ERR(get_file_offset(&b->delta_start, file, b->pool));
7422 /* Cleanup in case something goes wrong. */
7423 apr_pool_cleanup_register(b->pool, b, rep_write_cleanup,
7424 apr_pool_cleanup_null);
7426 /* Prepare to write the svndiff data. */
7427 svn_txdelta_to_svndiff3(&wh,
7431 SVN_DELTA_COMPRESSION_LEVEL_DEFAULT,
7434 b->delta_stream = svn_txdelta_target_push(wh, whb, source, b->pool);
7438 return SVN_NO_ERROR;
7441 /* For the hash REP->SHA1, try to find an already existing representation
7442 in FS and return it in *OUT_REP. If no such representation exists or
7443 if rep sharing has been disabled for FS, NULL will be returned. Since
7444 there may be new duplicate representations within the same uncommitted
7445 revision, those can be passed in REPS_HASH (maps a sha1 digest onto
7446 representation_t*), otherwise pass in NULL for REPS_HASH.
7447 POOL will be used for allocations. The lifetime of the returned rep is
7448 limited by both, POOL and REP lifetime.
7450 static svn_error_t *
7451 get_shared_rep(representation_t **old_rep,
7453 representation_t *rep,
7454 apr_hash_t *reps_hash,
7458 fs_fs_data_t *ffd = fs->fsap_data;
7460 /* Return NULL, if rep sharing has been disabled. */
7462 if (!ffd->rep_sharing_allowed)
7463 return SVN_NO_ERROR;
7465 /* Check and see if we already have a representation somewhere that's
7466 identical to the one we just wrote out. Start with the hash lookup
7467 because it is cheepest. */
7469 *old_rep = apr_hash_get(reps_hash,
7470 rep->sha1_checksum->digest,
7471 APR_SHA1_DIGESTSIZE);
7473 /* If we haven't found anything yet, try harder and consult our DB. */
7474 if (*old_rep == NULL)
7476 err = svn_fs_fs__get_rep_reference(old_rep, fs, rep->sha1_checksum,
7478 /* ### Other error codes that we shouldn't mask out? */
7479 if (err == SVN_NO_ERROR)
7482 SVN_ERR(verify_walker(*old_rep, NULL, fs, pool));
7484 else if (err->apr_err == SVN_ERR_FS_CORRUPT
7485 || SVN_ERROR_IN_CATEGORY(err->apr_err,
7486 SVN_ERR_MALFUNC_CATEGORY_START))
7488 /* Fatal error; don't mask it.
7490 In particular, this block is triggered when the rep-cache refers
7491 to revisions in the future. We signal that as a corruption situation
7492 since, once those revisions are less than youngest (because of more
7493 commits), the rep-cache would be invalid.
7499 /* Something's wrong with the rep-sharing index. We can continue
7500 without rep-sharing, but warn.
7502 (fs->warning)(fs->warning_baton, err);
7503 svn_error_clear(err);
7508 /* look for intra-revision matches (usually data reps but not limited
7509 to them in case props happen to look like some data rep)
7511 if (*old_rep == NULL && rep->txn_id)
7513 svn_node_kind_t kind;
7514 const char *file_name
7515 = path_txn_sha1(fs, rep->txn_id, rep->sha1_checksum, pool);
7517 /* in our txn, is there a rep file named with the wanted SHA1?
7518 If so, read it and use that rep.
7520 SVN_ERR(svn_io_check_path(file_name, &kind, pool));
7521 if (kind == svn_node_file)
7523 svn_stringbuf_t *rep_string;
7524 SVN_ERR(svn_stringbuf_from_file2(&rep_string, file_name, pool));
7525 SVN_ERR(read_rep_offsets_body(old_rep, rep_string->data,
7526 rep->txn_id, FALSE, pool));
7530 /* Add information that is missing in the cached data. */
7533 /* Use the old rep for this content. */
7534 (*old_rep)->md5_checksum = rep->md5_checksum;
7535 (*old_rep)->uniquifier = rep->uniquifier;
7538 return SVN_NO_ERROR;
7541 /* Close handler for the representation write stream. BATON is a
7542 rep_write_baton. Writes out a new node-rev that correctly
7543 references the representation we just finished writing. */
7544 static svn_error_t *
7545 rep_write_contents_close(void *baton)
7547 struct rep_write_baton *b = baton;
7548 const char *unique_suffix;
7549 representation_t *rep;
7550 representation_t *old_rep;
7553 rep = apr_pcalloc(b->parent_pool, sizeof(*rep));
7554 rep->offset = b->rep_offset;
7556 /* Close our delta stream so the last bits of svndiff are written
7558 if (b->delta_stream)
7559 SVN_ERR(svn_stream_close(b->delta_stream));
7561 /* Determine the length of the svndiff data. */
7562 SVN_ERR(get_file_offset(&offset, b->file, b->pool));
7563 rep->size = offset - b->delta_start;
7565 /* Fill in the rest of the representation field. */
7566 rep->expanded_size = b->rep_size;
7567 rep->txn_id = svn_fs_fs__id_txn_id(b->noderev->id);
7568 SVN_ERR(get_new_txn_node_id(&unique_suffix, b->fs, rep->txn_id, b->pool));
7569 rep->uniquifier = apr_psprintf(b->parent_pool, "%s/%s", rep->txn_id,
7571 rep->revision = SVN_INVALID_REVNUM;
7573 /* Finalize the checksum. */
7574 SVN_ERR(svn_checksum_final(&rep->md5_checksum, b->md5_checksum_ctx,
7576 SVN_ERR(svn_checksum_final(&rep->sha1_checksum, b->sha1_checksum_ctx,
7579 /* Check and see if we already have a representation somewhere that's
7580 identical to the one we just wrote out. */
7581 SVN_ERR(get_shared_rep(&old_rep, b->fs, rep, NULL, b->parent_pool));
7585 /* We need to erase from the protorev the data we just wrote. */
7586 SVN_ERR(svn_io_file_trunc(b->file, b->rep_offset, b->pool));
7588 /* Use the old rep for this content. */
7589 b->noderev->data_rep = old_rep;
7593 /* Write out our cosmetic end marker. */
7594 SVN_ERR(svn_stream_puts(b->rep_stream, "ENDREP\n"));
7596 b->noderev->data_rep = rep;
7599 /* Remove cleanup callback. */
7600 apr_pool_cleanup_kill(b->pool, b, rep_write_cleanup);
7602 /* Write out the new node-rev information. */
7603 SVN_ERR(svn_fs_fs__put_node_revision(b->fs, b->noderev->id, b->noderev, FALSE,
7606 SVN_ERR(store_sha1_rep_mapping(b->fs, b->noderev, b->pool));
7608 SVN_ERR(svn_io_file_close(b->file, b->pool));
7609 SVN_ERR(unlock_proto_rev(b->fs, rep->txn_id, b->lockcookie, b->pool));
7610 svn_pool_destroy(b->pool);
7612 return SVN_NO_ERROR;
7615 /* Store a writable stream in *CONTENTS_P that will receive all data
7616 written and store it as the file data representation referenced by
7617 NODEREV in filesystem FS. Perform temporary allocations in
7618 POOL. Only appropriate for file data, not props or directory
7620 static svn_error_t *
7621 set_representation(svn_stream_t **contents_p,
7623 node_revision_t *noderev,
7626 struct rep_write_baton *wb;
7628 if (! svn_fs_fs__id_txn_id(noderev->id))
7629 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
7630 _("Attempted to write to non-transaction '%s'"),
7631 svn_fs_fs__id_unparse(noderev->id, pool)->data);
7633 SVN_ERR(rep_write_get_baton(&wb, fs, noderev, pool));
7635 *contents_p = svn_stream_create(wb, pool);
7636 svn_stream_set_write(*contents_p, rep_write_contents);
7637 svn_stream_set_close(*contents_p, rep_write_contents_close);
7639 return SVN_NO_ERROR;
7643 svn_fs_fs__set_contents(svn_stream_t **stream,
7645 node_revision_t *noderev,
7648 if (noderev->kind != svn_node_file)
7649 return svn_error_create(SVN_ERR_FS_NOT_FILE, NULL,
7650 _("Can't set text contents of a directory"));
7652 return set_representation(stream, fs, noderev, pool);
7656 svn_fs_fs__create_successor(const svn_fs_id_t **new_id_p,
7658 const svn_fs_id_t *old_idp,
7659 node_revision_t *new_noderev,
7660 const char *copy_id,
7664 const svn_fs_id_t *id;
7667 copy_id = svn_fs_fs__id_copy_id(old_idp);
7668 id = svn_fs_fs__id_txn_create(svn_fs_fs__id_node_id(old_idp), copy_id,
7671 new_noderev->id = id;
7673 if (! new_noderev->copyroot_path)
7675 new_noderev->copyroot_path = apr_pstrdup(pool,
7676 new_noderev->created_path);
7677 new_noderev->copyroot_rev = svn_fs_fs__id_rev(new_noderev->id);
7680 SVN_ERR(svn_fs_fs__put_node_revision(fs, new_noderev->id, new_noderev, FALSE,
7685 return SVN_NO_ERROR;
7689 svn_fs_fs__set_proplist(svn_fs_t *fs,
7690 node_revision_t *noderev,
7691 apr_hash_t *proplist,
7694 const char *filename = path_txn_node_props(fs, noderev->id, pool);
7698 /* Dump the property list to the mutable property file. */
7699 SVN_ERR(svn_io_file_open(&file, filename,
7700 APR_WRITE | APR_CREATE | APR_TRUNCATE
7701 | APR_BUFFERED, APR_OS_DEFAULT, pool));
7702 out = svn_stream_from_aprfile2(file, TRUE, pool);
7703 SVN_ERR(svn_hash_write2(proplist, out, SVN_HASH_TERMINATOR, pool));
7704 SVN_ERR(svn_io_file_close(file, pool));
7706 /* Mark the node-rev's prop rep as mutable, if not already done. */
7707 if (!noderev->prop_rep || !noderev->prop_rep->txn_id)
7709 noderev->prop_rep = apr_pcalloc(pool, sizeof(*noderev->prop_rep));
7710 noderev->prop_rep->txn_id = svn_fs_fs__id_txn_id(noderev->id);
7711 SVN_ERR(svn_fs_fs__put_node_revision(fs, noderev->id, noderev, FALSE, pool));
7714 return SVN_NO_ERROR;
7717 /* Read the 'current' file for filesystem FS and store the next
7718 available node id in *NODE_ID, and the next available copy id in
7719 *COPY_ID. Allocations are performed from POOL. */
7720 static svn_error_t *
7721 get_next_revision_ids(const char **node_id,
7722 const char **copy_id,
7728 svn_stringbuf_t *content;
7730 SVN_ERR(read_content(&content, svn_fs_fs__path_current(fs, pool), pool));
7731 buf = content->data;
7733 str = svn_cstring_tokenize(" ", &buf);
7735 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
7736 _("Corrupt 'current' file"));
7738 str = svn_cstring_tokenize(" ", &buf);
7740 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
7741 _("Corrupt 'current' file"));
7743 *node_id = apr_pstrdup(pool, str);
7745 str = svn_cstring_tokenize(" \n", &buf);
7747 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
7748 _("Corrupt 'current' file"));
7750 *copy_id = apr_pstrdup(pool, str);
7752 return SVN_NO_ERROR;
7755 /* This baton is used by the stream created for write_hash_rep. */
7756 struct write_hash_baton
7758 svn_stream_t *stream;
7762 svn_checksum_ctx_t *md5_ctx;
7763 svn_checksum_ctx_t *sha1_ctx;
7766 /* The handler for the write_hash_rep stream. BATON is a
7767 write_hash_baton, DATA has the data to write and *LEN is the number
7768 of bytes to write. */
7769 static svn_error_t *
7770 write_hash_handler(void *baton,
7774 struct write_hash_baton *whb = baton;
7776 SVN_ERR(svn_checksum_update(whb->md5_ctx, data, *len));
7777 SVN_ERR(svn_checksum_update(whb->sha1_ctx, data, *len));
7779 SVN_ERR(svn_stream_write(whb->stream, data, len));
7782 return SVN_NO_ERROR;
7785 /* Write out the hash HASH as a text representation to file FILE. In
7786 the process, record position, the total size of the dump and MD5 as
7787 well as SHA1 in REP. If rep sharing has been enabled and REPS_HASH
7788 is not NULL, it will be used in addition to the on-disk cache to find
7789 earlier reps with the same content. When such existing reps can be
7790 found, we will truncate the one just written from the file and return
7791 the existing rep. Perform temporary allocations in POOL. */
7792 static svn_error_t *
7793 write_hash_rep(representation_t *rep,
7797 apr_hash_t *reps_hash,
7800 svn_stream_t *stream;
7801 struct write_hash_baton *whb;
7802 representation_t *old_rep;
7804 SVN_ERR(get_file_offset(&rep->offset, file, pool));
7806 whb = apr_pcalloc(pool, sizeof(*whb));
7808 whb->stream = svn_stream_from_aprfile2(file, TRUE, pool);
7810 whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool);
7811 whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool);
7813 stream = svn_stream_create(whb, pool);
7814 svn_stream_set_write(stream, write_hash_handler);
7816 SVN_ERR(svn_stream_puts(whb->stream, "PLAIN\n"));
7818 SVN_ERR(svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, pool));
7820 /* Store the results. */
7821 SVN_ERR(svn_checksum_final(&rep->md5_checksum, whb->md5_ctx, pool));
7822 SVN_ERR(svn_checksum_final(&rep->sha1_checksum, whb->sha1_ctx, pool));
7824 /* Check and see if we already have a representation somewhere that's
7825 identical to the one we just wrote out. */
7826 SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, pool));
7830 /* We need to erase from the protorev the data we just wrote. */
7831 SVN_ERR(svn_io_file_trunc(file, rep->offset, pool));
7833 /* Use the old rep for this content. */
7834 memcpy(rep, old_rep, sizeof (*rep));
7838 /* Write out our cosmetic end marker. */
7839 SVN_ERR(svn_stream_puts(whb->stream, "ENDREP\n"));
7841 /* update the representation */
7842 rep->size = whb->size;
7843 rep->expanded_size = 0;
7846 return SVN_NO_ERROR;
7849 /* Write out the hash HASH pertaining to the NODEREV in FS as a deltified
7850 text representation to file FILE. In the process, record the total size
7851 and the md5 digest in REP. If rep sharing has been enabled and REPS_HASH
7852 is not NULL, it will be used in addition to the on-disk cache to find
7853 earlier reps with the same content. When such existing reps can be found,
7854 we will truncate the one just written from the file and return the existing
7855 rep. If PROPS is set, assume that we want to a props representation as
7856 the base for our delta. Perform temporary allocations in POOL. */
7857 static svn_error_t *
7858 write_hash_delta_rep(representation_t *rep,
7862 node_revision_t *noderev,
7863 apr_hash_t *reps_hash,
7864 svn_boolean_t props,
7867 svn_txdelta_window_handler_t diff_wh;
7870 svn_stream_t *file_stream;
7871 svn_stream_t *stream;
7872 representation_t *base_rep;
7873 representation_t *old_rep;
7874 svn_stream_t *source;
7877 apr_off_t rep_end = 0;
7878 apr_off_t delta_start = 0;
7880 struct write_hash_baton *whb;
7881 fs_fs_data_t *ffd = fs->fsap_data;
7882 int diff_version = ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT ? 1 : 0;
7884 /* Get the base for this delta. */
7885 SVN_ERR(choose_delta_base(&base_rep, fs, noderev, props, pool));
7886 SVN_ERR(read_representation(&source, fs, base_rep, pool));
7888 SVN_ERR(get_file_offset(&rep->offset, file, pool));
7890 /* Write out the rep header. */
7893 header = apr_psprintf(pool, REP_DELTA " %ld %" APR_OFF_T_FMT " %"
7894 SVN_FILESIZE_T_FMT "\n",
7895 base_rep->revision, base_rep->offset,
7900 header = REP_DELTA "\n";
7902 SVN_ERR(svn_io_file_write_full(file, header, strlen(header), NULL,
7905 SVN_ERR(get_file_offset(&delta_start, file, pool));
7906 file_stream = svn_stream_from_aprfile2(file, TRUE, pool);
7908 /* Prepare to write the svndiff data. */
7909 svn_txdelta_to_svndiff3(&diff_wh,
7913 SVN_DELTA_COMPRESSION_LEVEL_DEFAULT,
7916 whb = apr_pcalloc(pool, sizeof(*whb));
7917 whb->stream = svn_txdelta_target_push(diff_wh, diff_whb, source, pool);
7919 whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool);
7920 whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool);
7922 /* serialize the hash */
7923 stream = svn_stream_create(whb, pool);
7924 svn_stream_set_write(stream, write_hash_handler);
7926 SVN_ERR(svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, pool));
7927 SVN_ERR(svn_stream_close(whb->stream));
7929 /* Store the results. */
7930 SVN_ERR(svn_checksum_final(&rep->md5_checksum, whb->md5_ctx, pool));
7931 SVN_ERR(svn_checksum_final(&rep->sha1_checksum, whb->sha1_ctx, pool));
7933 /* Check and see if we already have a representation somewhere that's
7934 identical to the one we just wrote out. */
7935 SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, pool));
7939 /* We need to erase from the protorev the data we just wrote. */
7940 SVN_ERR(svn_io_file_trunc(file, rep->offset, pool));
7942 /* Use the old rep for this content. */
7943 memcpy(rep, old_rep, sizeof (*rep));
7947 /* Write out our cosmetic end marker. */
7948 SVN_ERR(get_file_offset(&rep_end, file, pool));
7949 SVN_ERR(svn_stream_puts(file_stream, "ENDREP\n"));
7951 /* update the representation */
7952 rep->expanded_size = whb->size;
7953 rep->size = rep_end - delta_start;
7956 return SVN_NO_ERROR;
7959 /* Sanity check ROOT_NODEREV, a candidate for being the root node-revision
7960 of (not yet committed) revision REV in FS. Use POOL for temporary
7963 If you change this function, consider updating svn_fs_fs__verify() too.
7965 static svn_error_t *
7966 validate_root_noderev(svn_fs_t *fs,
7967 node_revision_t *root_noderev,
7971 svn_revnum_t head_revnum = rev-1;
7972 int head_predecessor_count;
7974 SVN_ERR_ASSERT(rev > 0);
7976 /* Compute HEAD_PREDECESSOR_COUNT. */
7978 svn_fs_root_t *head_revision;
7979 const svn_fs_id_t *head_root_id;
7980 node_revision_t *head_root_noderev;
7982 /* Get /@HEAD's noderev. */
7983 SVN_ERR(svn_fs_fs__revision_root(&head_revision, fs, head_revnum, pool));
7984 SVN_ERR(svn_fs_fs__node_id(&head_root_id, head_revision, "/", pool));
7985 SVN_ERR(svn_fs_fs__get_node_revision(&head_root_noderev, fs, head_root_id,
7988 head_predecessor_count = head_root_noderev->predecessor_count;
7991 /* Check that the root noderev's predecessor count equals REV.
7993 This kind of corruption was seen on svn.apache.org (both on
7994 the root noderev and on other fspaths' noderevs); see
7997 Normally (rev == root_noderev->predecessor_count), but here we
7998 use a more roundabout check that should only trigger on new instances
7999 of the corruption, rather then trigger on each and every new commit
8000 to a repository that has triggered the bug somewhere in its root
8003 if (root_noderev->predecessor_count != -1
8004 && (root_noderev->predecessor_count - head_predecessor_count)
8005 != (rev - head_revnum))
8007 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
8008 _("predecessor count for "
8009 "the root node-revision is wrong: "
8010 "found (%d+%ld != %d), committing r%ld"),
8011 head_predecessor_count,
8012 rev - head_revnum, /* This is equal to 1. */
8013 root_noderev->predecessor_count,
8017 return SVN_NO_ERROR;
8020 /* Copy a node-revision specified by id ID in fileystem FS from a
8021 transaction into the proto-rev-file FILE. Set *NEW_ID_P to a
8022 pointer to the new node-id which will be allocated in POOL.
8023 If this is a directory, copy all children as well.
8025 START_NODE_ID and START_COPY_ID are
8026 the first available node and copy ids for this filesystem, for older
8029 REV is the revision number that this proto-rev-file will represent.
8031 INITIAL_OFFSET is the offset of the proto-rev-file on entry to
8034 If REPS_TO_CACHE is not NULL, append to it a copy (allocated in
8035 REPS_POOL) of each data rep that is new in this revision.
8037 If REPS_HASH is not NULL, append copies (allocated in REPS_POOL)
8038 of the representations of each property rep that is new in this
8041 AT_ROOT is true if the node revision being written is the root
8042 node-revision. It is only controls additional sanity checking
8045 Temporary allocations are also from POOL. */
8046 static svn_error_t *
8047 write_final_rev(const svn_fs_id_t **new_id_p,
8051 const svn_fs_id_t *id,
8052 const char *start_node_id,
8053 const char *start_copy_id,
8054 apr_off_t initial_offset,
8055 apr_array_header_t *reps_to_cache,
8056 apr_hash_t *reps_hash,
8057 apr_pool_t *reps_pool,
8058 svn_boolean_t at_root,
8061 node_revision_t *noderev;
8062 apr_off_t my_offset;
8063 char my_node_id_buf[MAX_KEY_SIZE + 2];
8064 char my_copy_id_buf[MAX_KEY_SIZE + 2];
8065 const svn_fs_id_t *new_id;
8066 const char *node_id, *copy_id, *my_node_id, *my_copy_id;
8067 fs_fs_data_t *ffd = fs->fsap_data;
8071 /* Check to see if this is a transaction node. */
8072 if (! svn_fs_fs__id_txn_id(id))
8073 return SVN_NO_ERROR;
8075 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, id, pool));
8077 if (noderev->kind == svn_node_dir)
8079 apr_pool_t *subpool;
8080 apr_hash_t *entries, *str_entries;
8081 apr_array_header_t *sorted_entries;
8084 /* This is a directory. Write out all the children first. */
8085 subpool = svn_pool_create(pool);
8087 SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, noderev, pool));
8088 /* For the sake of the repository administrator sort the entries
8089 so that the final file is deterministic and repeatable,
8090 however the rest of the FSFS code doesn't require any
8091 particular order here. */
8092 sorted_entries = svn_sort__hash(entries, svn_sort_compare_items_lexically,
8094 for (i = 0; i < sorted_entries->nelts; ++i)
8096 svn_fs_dirent_t *dirent = APR_ARRAY_IDX(sorted_entries, i,
8097 svn_sort__item_t).value;
8099 svn_pool_clear(subpool);
8100 SVN_ERR(write_final_rev(&new_id, file, rev, fs, dirent->id,
8101 start_node_id, start_copy_id, initial_offset,
8102 reps_to_cache, reps_hash, reps_pool, FALSE,
8104 if (new_id && (svn_fs_fs__id_rev(new_id) == rev))
8105 dirent->id = svn_fs_fs__id_copy(new_id, pool);
8107 svn_pool_destroy(subpool);
8109 if (noderev->data_rep && noderev->data_rep->txn_id)
8111 /* Write out the contents of this directory as a text rep. */
8112 SVN_ERR(unparse_dir_entries(&str_entries, entries, pool));
8114 noderev->data_rep->txn_id = NULL;
8115 noderev->data_rep->revision = rev;
8117 if (ffd->deltify_directories)
8118 SVN_ERR(write_hash_delta_rep(noderev->data_rep, file,
8119 str_entries, fs, noderev, NULL,
8122 SVN_ERR(write_hash_rep(noderev->data_rep, file, str_entries,
8128 /* This is a file. We should make sure the data rep, if it
8129 exists in a "this" state, gets rewritten to our new revision
8132 if (noderev->data_rep && noderev->data_rep->txn_id)
8134 noderev->data_rep->txn_id = NULL;
8135 noderev->data_rep->revision = rev;
8137 /* See issue 3845. Some unknown mechanism caused the
8138 protorev file to get truncated, so check for that
8140 if (noderev->data_rep->offset + noderev->data_rep->size
8142 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
8143 _("Truncated protorev file detected"));
8147 /* Fix up the property reps. */
8148 if (noderev->prop_rep && noderev->prop_rep->txn_id)
8150 apr_hash_t *proplist;
8151 SVN_ERR(svn_fs_fs__get_proplist(&proplist, fs, noderev, pool));
8153 noderev->prop_rep->txn_id = NULL;
8154 noderev->prop_rep->revision = rev;
8156 if (ffd->deltify_properties)
8157 SVN_ERR(write_hash_delta_rep(noderev->prop_rep, file,
8158 proplist, fs, noderev, reps_hash,
8161 SVN_ERR(write_hash_rep(noderev->prop_rep, file, proplist,
8162 fs, reps_hash, pool));
8166 /* Convert our temporary ID into a permanent revision one. */
8167 SVN_ERR(get_file_offset(&my_offset, file, pool));
8169 node_id = svn_fs_fs__id_node_id(noderev->id);
8170 if (*node_id == '_')
8172 if (ffd->format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT)
8173 my_node_id = apr_psprintf(pool, "%s-%ld", node_id + 1, rev);
8176 svn_fs_fs__add_keys(start_node_id, node_id + 1, my_node_id_buf);
8177 my_node_id = my_node_id_buf;
8181 my_node_id = node_id;
8183 copy_id = svn_fs_fs__id_copy_id(noderev->id);
8184 if (*copy_id == '_')
8186 if (ffd->format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT)
8187 my_copy_id = apr_psprintf(pool, "%s-%ld", copy_id + 1, rev);
8190 svn_fs_fs__add_keys(start_copy_id, copy_id + 1, my_copy_id_buf);
8191 my_copy_id = my_copy_id_buf;
8195 my_copy_id = copy_id;
8197 if (noderev->copyroot_rev == SVN_INVALID_REVNUM)
8198 noderev->copyroot_rev = rev;
8200 new_id = svn_fs_fs__id_rev_create(my_node_id, my_copy_id, rev, my_offset,
8203 noderev->id = new_id;
8205 if (ffd->rep_sharing_allowed)
8207 /* Save the data representation's hash in the rep cache. */
8208 if ( noderev->data_rep && noderev->kind == svn_node_file
8209 && noderev->data_rep->revision == rev)
8211 SVN_ERR_ASSERT(reps_to_cache && reps_pool);
8212 APR_ARRAY_PUSH(reps_to_cache, representation_t *)
8213 = svn_fs_fs__rep_copy(noderev->data_rep, reps_pool);
8216 if (noderev->prop_rep && noderev->prop_rep->revision == rev)
8218 /* Add new property reps to hash and on-disk cache. */
8219 representation_t *copy
8220 = svn_fs_fs__rep_copy(noderev->prop_rep, reps_pool);
8222 SVN_ERR_ASSERT(reps_to_cache && reps_pool);
8223 APR_ARRAY_PUSH(reps_to_cache, representation_t *) = copy;
8225 apr_hash_set(reps_hash,
8226 copy->sha1_checksum->digest,
8227 APR_SHA1_DIGESTSIZE,
8232 /* don't serialize SHA1 for dirs to disk (waste of space) */
8233 if (noderev->data_rep && noderev->kind == svn_node_dir)
8234 noderev->data_rep->sha1_checksum = NULL;
8236 /* don't serialize SHA1 for props to disk (waste of space) */
8237 if (noderev->prop_rep)
8238 noderev->prop_rep->sha1_checksum = NULL;
8240 /* Workaround issue #4031: is-fresh-txn-root in revision files. */
8241 noderev->is_fresh_txn_root = FALSE;
8243 /* Write out our new node-revision. */
8245 SVN_ERR(validate_root_noderev(fs, noderev, rev, pool));
8247 SVN_ERR(svn_fs_fs__write_noderev(svn_stream_from_aprfile2(file, TRUE, pool),
8248 noderev, ffd->format,
8249 svn_fs_fs__fs_supports_mergeinfo(fs),
8252 /* Return our ID that references the revision file. */
8253 *new_id_p = noderev->id;
8255 return SVN_NO_ERROR;
8258 /* Write the changed path info from transaction TXN_ID in filesystem
8259 FS to the permanent rev-file FILE. *OFFSET_P is set the to offset
8260 in the file of the beginning of this information. Perform
8261 temporary allocations in POOL. */
8262 static svn_error_t *
8263 write_final_changed_path_info(apr_off_t *offset_p,
8269 apr_hash_t *changed_paths;
8271 apr_pool_t *iterpool = svn_pool_create(pool);
8272 fs_fs_data_t *ffd = fs->fsap_data;
8273 svn_boolean_t include_node_kinds =
8274 ffd->format >= SVN_FS_FS__MIN_KIND_IN_CHANGED_FORMAT;
8275 apr_array_header_t *sorted_changed_paths;
8278 SVN_ERR(get_file_offset(&offset, file, pool));
8280 SVN_ERR(svn_fs_fs__txn_changes_fetch(&changed_paths, fs, txn_id, pool));
8281 /* For the sake of the repository administrator sort the changes so
8282 that the final file is deterministic and repeatable, however the
8283 rest of the FSFS code doesn't require any particular order here. */
8284 sorted_changed_paths = svn_sort__hash(changed_paths,
8285 svn_sort_compare_items_lexically, pool);
8287 /* Iterate through the changed paths one at a time, and convert the
8288 temporary node-id into a permanent one for each change entry. */
8289 for (i = 0; i < sorted_changed_paths->nelts; ++i)
8291 node_revision_t *noderev;
8292 const svn_fs_id_t *id;
8293 svn_fs_path_change2_t *change;
8296 svn_pool_clear(iterpool);
8298 change = APR_ARRAY_IDX(sorted_changed_paths, i, svn_sort__item_t).value;
8299 path = APR_ARRAY_IDX(sorted_changed_paths, i, svn_sort__item_t).key;
8301 id = change->node_rev_id;
8303 /* If this was a delete of a mutable node, then it is OK to
8304 leave the change entry pointing to the non-existent temporary
8305 node, since it will never be used. */
8306 if ((change->change_kind != svn_fs_path_change_delete) &&
8307 (! svn_fs_fs__id_txn_id(id)))
8309 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, id, iterpool));
8311 /* noderev has the permanent node-id at this point, so we just
8312 substitute it for the temporary one. */
8313 change->node_rev_id = noderev->id;
8316 /* Write out the new entry into the final rev-file. */
8317 SVN_ERR(write_change_entry(file, path, change, include_node_kinds,
8321 svn_pool_destroy(iterpool);
8325 return SVN_NO_ERROR;
8328 /* Atomically update the 'current' file to hold the specifed REV,
8329 NEXT_NODE_ID, and NEXT_COPY_ID. (The two next-ID parameters are
8330 ignored and may be NULL if the FS format does not use them.)
8331 Perform temporary allocations in POOL. */
8332 static svn_error_t *
8333 write_current(svn_fs_t *fs, svn_revnum_t rev, const char *next_node_id,
8334 const char *next_copy_id, apr_pool_t *pool)
8337 const char *tmp_name, *name;
8338 fs_fs_data_t *ffd = fs->fsap_data;
8340 /* Now we can just write out this line. */
8341 if (ffd->format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT)
8342 buf = apr_psprintf(pool, "%ld\n", rev);
8344 buf = apr_psprintf(pool, "%ld %s %s\n", rev, next_node_id, next_copy_id);
8346 name = svn_fs_fs__path_current(fs, pool);
8347 SVN_ERR(svn_io_write_unique(&tmp_name,
8348 svn_dirent_dirname(name, pool),
8350 svn_io_file_del_none, pool));
8352 return move_into_place(tmp_name, name, name, pool);
8355 /* Open a new svn_fs_t handle to FS, set that handle's concept of "current
8356 youngest revision" to NEW_REV, and call svn_fs_fs__verify_root() on
8357 NEW_REV's revision root.
8359 Intended to be called as the very last step in a commit before 'current'
8360 is bumped. This implies that we are holding the write lock. */
8361 static svn_error_t *
8362 verify_as_revision_before_current_plus_plus(svn_fs_t *fs,
8363 svn_revnum_t new_rev,
8367 fs_fs_data_t *ffd = fs->fsap_data;
8368 svn_fs_t *ft; /* fs++ == ft */
8369 svn_fs_root_t *root;
8370 fs_fs_data_t *ft_ffd;
8371 apr_hash_t *fs_config;
8373 SVN_ERR_ASSERT(ffd->svn_fs_open_);
8375 /* make sure FT does not simply return data cached by other instances
8376 * but actually retrieves it from disk at least once.
8378 fs_config = apr_hash_make(pool);
8379 svn_hash_sets(fs_config, SVN_FS_CONFIG_FSFS_CACHE_NS,
8380 svn_uuid_generate(pool));
8381 SVN_ERR(ffd->svn_fs_open_(&ft, fs->path,
8384 ft_ffd = ft->fsap_data;
8385 /* Don't let FT consult rep-cache.db, either. */
8386 ft_ffd->rep_sharing_allowed = FALSE;
8389 ft_ffd->youngest_rev_cache = new_rev;
8391 SVN_ERR(svn_fs_fs__revision_root(&root, ft, new_rev, pool));
8392 SVN_ERR_ASSERT(root->is_txn_root == FALSE && root->rev == new_rev);
8393 SVN_ERR_ASSERT(ft_ffd->youngest_rev_cache == new_rev);
8394 SVN_ERR(svn_fs_fs__verify_root(root, pool));
8395 #endif /* SVN_DEBUG */
8397 return SVN_NO_ERROR;
8400 /* Update the 'current' file to hold the correct next node and copy_ids
8401 from transaction TXN_ID in filesystem FS. The current revision is
8402 set to REV. Perform temporary allocations in POOL. */
8403 static svn_error_t *
8404 write_final_current(svn_fs_t *fs,
8407 const char *start_node_id,
8408 const char *start_copy_id,
8411 const char *txn_node_id, *txn_copy_id;
8412 char new_node_id[MAX_KEY_SIZE + 2];
8413 char new_copy_id[MAX_KEY_SIZE + 2];
8414 fs_fs_data_t *ffd = fs->fsap_data;
8416 if (ffd->format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT)
8417 return write_current(fs, rev, NULL, NULL, pool);
8419 /* To find the next available ids, we add the id that used to be in
8420 the 'current' file, to the next ids from the transaction file. */
8421 SVN_ERR(read_next_ids(&txn_node_id, &txn_copy_id, fs, txn_id, pool));
8423 svn_fs_fs__add_keys(start_node_id, txn_node_id, new_node_id);
8424 svn_fs_fs__add_keys(start_copy_id, txn_copy_id, new_copy_id);
8426 return write_current(fs, rev, new_node_id, new_copy_id, pool);
8429 /* Verify that the user registed with FS has all the locks necessary to
8430 permit all the changes associate with TXN_NAME.
8431 The FS write lock is assumed to be held by the caller. */
8432 static svn_error_t *
8433 verify_locks(svn_fs_t *fs,
8434 const char *txn_name,
8437 apr_pool_t *subpool = svn_pool_create(pool);
8438 apr_hash_t *changes;
8439 apr_hash_index_t *hi;
8440 apr_array_header_t *changed_paths;
8441 svn_stringbuf_t *last_recursed = NULL;
8444 /* Fetch the changes for this transaction. */
8445 SVN_ERR(svn_fs_fs__txn_changes_fetch(&changes, fs, txn_name, pool));
8447 /* Make an array of the changed paths, and sort them depth-first-ily. */
8448 changed_paths = apr_array_make(pool, apr_hash_count(changes) + 1,
8449 sizeof(const char *));
8450 for (hi = apr_hash_first(pool, changes); hi; hi = apr_hash_next(hi))
8451 APR_ARRAY_PUSH(changed_paths, const char *) = svn__apr_hash_index_key(hi);
8452 qsort(changed_paths->elts, changed_paths->nelts,
8453 changed_paths->elt_size, svn_sort_compare_paths);
8455 /* Now, traverse the array of changed paths, verify locks. Note
8456 that if we need to do a recursive verification a path, we'll skip
8457 over children of that path when we get to them. */
8458 for (i = 0; i < changed_paths->nelts; i++)
8461 svn_fs_path_change2_t *change;
8462 svn_boolean_t recurse = TRUE;
8464 svn_pool_clear(subpool);
8465 path = APR_ARRAY_IDX(changed_paths, i, const char *);
8467 /* If this path has already been verified as part of a recursive
8468 check of one of its parents, no need to do it again. */
8470 && svn_dirent_is_child(last_recursed->data, path, subpool))
8473 /* Fetch the change associated with our path. */
8474 change = svn_hash_gets(changes, path);
8476 /* What does it mean to succeed at lock verification for a given
8477 path? For an existing file or directory getting modified
8478 (text, props), it means we hold the lock on the file or
8479 directory. For paths being added or removed, we need to hold
8480 the locks for that path and any children of that path.
8482 WHEW! We have no reliable way to determine the node kind
8483 of deleted items, but fortunately we are going to do a
8484 recursive check on deleted paths regardless of their kind. */
8485 if (change->change_kind == svn_fs_path_change_modify)
8487 SVN_ERR(svn_fs_fs__allow_locked_operation(path, fs, recurse, TRUE,
8490 /* If we just did a recursive check, remember the path we
8491 checked (so children can be skipped). */
8494 if (! last_recursed)
8495 last_recursed = svn_stringbuf_create(path, pool);
8497 svn_stringbuf_set(last_recursed, path);
8500 svn_pool_destroy(subpool);
8501 return SVN_NO_ERROR;
8504 /* Baton used for commit_body below. */
8505 struct commit_baton {
8506 svn_revnum_t *new_rev_p;
8509 apr_array_header_t *reps_to_cache;
8510 apr_hash_t *reps_hash;
8511 apr_pool_t *reps_pool;
8514 /* The work-horse for svn_fs_fs__commit, called with the FS write lock.
8515 This implements the svn_fs_fs__with_write_lock() 'body' callback
8516 type. BATON is a 'struct commit_baton *'. */
8517 static svn_error_t *
8518 commit_body(void *baton, apr_pool_t *pool)
8520 struct commit_baton *cb = baton;
8521 fs_fs_data_t *ffd = cb->fs->fsap_data;
8522 const char *old_rev_filename, *rev_filename, *proto_filename;
8523 const char *revprop_filename, *final_revprop;
8524 const svn_fs_id_t *root_id, *new_root_id;
8525 const char *start_node_id = NULL, *start_copy_id = NULL;
8526 svn_revnum_t old_rev, new_rev;
8527 apr_file_t *proto_file;
8528 void *proto_file_lockcookie;
8529 apr_off_t initial_offset, changed_path_offset;
8531 apr_hash_t *txnprops;
8532 apr_array_header_t *txnprop_list;
8536 /* Get the current youngest revision. */
8537 SVN_ERR(svn_fs_fs__youngest_rev(&old_rev, cb->fs, pool));
8539 /* Check to make sure this transaction is based off the most recent
8541 if (cb->txn->base_rev != old_rev)
8542 return svn_error_create(SVN_ERR_FS_TXN_OUT_OF_DATE, NULL,
8543 _("Transaction out of date"));
8545 /* Locks may have been added (or stolen) between the calling of
8546 previous svn_fs.h functions and svn_fs_commit_txn(), so we need
8547 to re-examine every changed-path in the txn and re-verify all
8548 discovered locks. */
8549 SVN_ERR(verify_locks(cb->fs, cb->txn->id, pool));
8551 /* Get the next node_id and copy_id to use. */
8552 if (ffd->format < SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT)
8553 SVN_ERR(get_next_revision_ids(&start_node_id, &start_copy_id, cb->fs,
8556 /* We are going to be one better than this puny old revision. */
8557 new_rev = old_rev + 1;
8559 /* Get a write handle on the proto revision file. */
8560 SVN_ERR(get_writable_proto_rev(&proto_file, &proto_file_lockcookie,
8561 cb->fs, cb->txn->id, pool));
8562 SVN_ERR(get_file_offset(&initial_offset, proto_file, pool));
8564 /* Write out all the node-revisions and directory contents. */
8565 root_id = svn_fs_fs__id_txn_create("0", "0", cb->txn->id, pool);
8566 SVN_ERR(write_final_rev(&new_root_id, proto_file, new_rev, cb->fs, root_id,
8567 start_node_id, start_copy_id, initial_offset,
8568 cb->reps_to_cache, cb->reps_hash, cb->reps_pool,
8571 /* Write the changed-path information. */
8572 SVN_ERR(write_final_changed_path_info(&changed_path_offset, proto_file,
8573 cb->fs, cb->txn->id, pool));
8575 /* Write the final line. */
8576 buf = apr_psprintf(pool, "\n%" APR_OFF_T_FMT " %" APR_OFF_T_FMT "\n",
8577 svn_fs_fs__id_offset(new_root_id),
8578 changed_path_offset);
8579 SVN_ERR(svn_io_file_write_full(proto_file, buf, strlen(buf), NULL,
8581 SVN_ERR(svn_io_file_flush_to_disk(proto_file, pool));
8582 SVN_ERR(svn_io_file_close(proto_file, pool));
8584 /* We don't unlock the prototype revision file immediately to avoid a
8585 race with another caller writing to the prototype revision file
8586 before we commit it. */
8588 /* Remove any temporary txn props representing 'flags'. */
8589 SVN_ERR(svn_fs_fs__txn_proplist(&txnprops, cb->txn, pool));
8590 txnprop_list = apr_array_make(pool, 3, sizeof(svn_prop_t));
8593 if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_OOD))
8595 prop.name = SVN_FS__PROP_TXN_CHECK_OOD;
8596 APR_ARRAY_PUSH(txnprop_list, svn_prop_t) = prop;
8599 if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_LOCKS))
8601 prop.name = SVN_FS__PROP_TXN_CHECK_LOCKS;
8602 APR_ARRAY_PUSH(txnprop_list, svn_prop_t) = prop;
8605 if (! apr_is_empty_array(txnprop_list))
8606 SVN_ERR(svn_fs_fs__change_txn_props(cb->txn, txnprop_list, pool));
8608 /* Create the shard for the rev and revprop file, if we're sharding and
8609 this is the first revision of a new shard. We don't care if this
8610 fails because the shard already existed for some reason. */
8611 if (ffd->max_files_per_dir && new_rev % ffd->max_files_per_dir == 0)
8613 /* Create the revs shard. */
8615 const char *new_dir = path_rev_shard(cb->fs, new_rev, pool);
8616 svn_error_t *err = svn_io_dir_make(new_dir, APR_OS_DEFAULT, pool);
8617 if (err && !APR_STATUS_IS_EEXIST(err->apr_err))
8618 return svn_error_trace(err);
8619 svn_error_clear(err);
8620 SVN_ERR(svn_io_copy_perms(svn_dirent_join(cb->fs->path,
8626 /* Create the revprops shard. */
8627 SVN_ERR_ASSERT(! is_packed_revprop(cb->fs, new_rev));
8629 const char *new_dir = path_revprops_shard(cb->fs, new_rev, pool);
8630 svn_error_t *err = svn_io_dir_make(new_dir, APR_OS_DEFAULT, pool);
8631 if (err && !APR_STATUS_IS_EEXIST(err->apr_err))
8632 return svn_error_trace(err);
8633 svn_error_clear(err);
8634 SVN_ERR(svn_io_copy_perms(svn_dirent_join(cb->fs->path,
8641 /* Move the finished rev file into place. */
8642 SVN_ERR(svn_fs_fs__path_rev_absolute(&old_rev_filename,
8643 cb->fs, old_rev, pool));
8644 rev_filename = path_rev(cb->fs, new_rev, pool);
8645 proto_filename = path_txn_proto_rev(cb->fs, cb->txn->id, pool);
8646 SVN_ERR(move_into_place(proto_filename, rev_filename, old_rev_filename,
8649 /* Now that we've moved the prototype revision file out of the way,
8650 we can unlock it (since further attempts to write to the file
8651 will fail as it no longer exists). We must do this so that we can
8652 remove the transaction directory later. */
8653 SVN_ERR(unlock_proto_rev(cb->fs, cb->txn->id, proto_file_lockcookie, pool));
8655 /* Update commit time to ensure that svn:date revprops remain ordered. */
8656 date.data = svn_time_to_cstring(apr_time_now(), pool);
8657 date.len = strlen(date.data);
8659 SVN_ERR(svn_fs_fs__change_txn_prop(cb->txn, SVN_PROP_REVISION_DATE,
8662 /* Move the revprops file into place. */
8663 SVN_ERR_ASSERT(! is_packed_revprop(cb->fs, new_rev));
8664 revprop_filename = path_txn_props(cb->fs, cb->txn->id, pool);
8665 final_revprop = path_revprops(cb->fs, new_rev, pool);
8666 SVN_ERR(move_into_place(revprop_filename, final_revprop,
8667 old_rev_filename, pool));
8669 /* Update the 'current' file. */
8670 SVN_ERR(verify_as_revision_before_current_plus_plus(cb->fs, new_rev, pool));
8671 SVN_ERR(write_final_current(cb->fs, cb->txn->id, new_rev, start_node_id,
8672 start_copy_id, pool));
8674 /* At this point the new revision is committed and globally visible
8675 so let the caller know it succeeded by giving it the new revision
8676 number, which fulfills svn_fs_commit_txn() contract. Any errors
8677 after this point do not change the fact that a new revision was
8679 *cb->new_rev_p = new_rev;
8681 ffd->youngest_rev_cache = new_rev;
8683 /* Remove this transaction directory. */
8684 SVN_ERR(svn_fs_fs__purge_txn(cb->fs, cb->txn->id, pool));
8686 return SVN_NO_ERROR;
8689 /* Add the representations in REPS_TO_CACHE (an array of representation_t *)
8690 * to the rep-cache database of FS. */
8691 static svn_error_t *
8692 write_reps_to_cache(svn_fs_t *fs,
8693 const apr_array_header_t *reps_to_cache,
8694 apr_pool_t *scratch_pool)
8698 for (i = 0; i < reps_to_cache->nelts; i++)
8700 representation_t *rep = APR_ARRAY_IDX(reps_to_cache, i, representation_t *);
8702 /* FALSE because we don't care if another parallel commit happened to
8703 * collide with us. (Non-parallel collisions will not be detected.) */
8704 SVN_ERR(svn_fs_fs__set_rep_reference(fs, rep, FALSE, scratch_pool));
8707 return SVN_NO_ERROR;
8711 svn_fs_fs__commit(svn_revnum_t *new_rev_p,
8716 struct commit_baton cb;
8717 fs_fs_data_t *ffd = fs->fsap_data;
8719 cb.new_rev_p = new_rev_p;
8723 if (ffd->rep_sharing_allowed)
8725 cb.reps_to_cache = apr_array_make(pool, 5, sizeof(representation_t *));
8726 cb.reps_hash = apr_hash_make(pool);
8727 cb.reps_pool = pool;
8731 cb.reps_to_cache = NULL;
8732 cb.reps_hash = NULL;
8733 cb.reps_pool = NULL;
8736 SVN_ERR(svn_fs_fs__with_write_lock(fs, commit_body, &cb, pool));
8738 /* At this point, *NEW_REV_P has been set, so errors below won't affect
8739 the success of the commit. (See svn_fs_commit_txn().) */
8741 if (ffd->rep_sharing_allowed)
8743 SVN_ERR(svn_fs_fs__open_rep_cache(fs, pool));
8745 /* Write new entries to the rep-sharing database.
8747 * We use an sqlite transaction to speed things up;
8748 * see <http://www.sqlite.org/faq.html#q19>.
8750 SVN_SQLITE__WITH_TXN(
8751 write_reps_to_cache(fs, cb.reps_to_cache, pool),
8755 return SVN_NO_ERROR;
8760 svn_fs_fs__reserve_copy_id(const char **copy_id_p,
8765 const char *cur_node_id, *cur_copy_id;
8769 /* First read in the current next-ids file. */
8770 SVN_ERR(read_next_ids(&cur_node_id, &cur_copy_id, fs, txn_id, pool));
8772 copy_id = apr_pcalloc(pool, strlen(cur_copy_id) + 2);
8774 len = strlen(cur_copy_id);
8775 svn_fs_fs__next_key(cur_copy_id, &len, copy_id);
8777 SVN_ERR(write_next_ids(fs, txn_id, cur_node_id, copy_id, pool));
8779 *copy_id_p = apr_pstrcat(pool, "_", cur_copy_id, (char *)NULL);
8781 return SVN_NO_ERROR;
8784 /* Write out the zeroth revision for filesystem FS. */
8785 static svn_error_t *
8786 write_revision_zero(svn_fs_t *fs)
8788 const char *path_revision_zero = path_rev(fs, 0, fs->pool);
8789 apr_hash_t *proplist;
8792 /* Write out a rev file for revision 0. */
8793 SVN_ERR(svn_io_file_create(path_revision_zero,
8794 "PLAIN\nEND\nENDREP\n"
8799 "2d2977d1c96f487abe4a1e202dd03b4e\n"
8801 "\n\n17 107\n", fs->pool));
8802 SVN_ERR(svn_io_set_file_read_only(path_revision_zero, FALSE, fs->pool));
8804 /* Set a date on revision 0. */
8805 date.data = svn_time_to_cstring(apr_time_now(), fs->pool);
8806 date.len = strlen(date.data);
8807 proplist = apr_hash_make(fs->pool);
8808 svn_hash_sets(proplist, SVN_PROP_REVISION_DATE, &date);
8809 return set_revision_proplist(fs, 0, proplist, fs->pool);
8813 svn_fs_fs__create(svn_fs_t *fs,
8817 int format = SVN_FS_FS__FORMAT_NUMBER;
8818 fs_fs_data_t *ffd = fs->fsap_data;
8820 fs->path = apr_pstrdup(pool, path);
8821 /* See if compatibility with older versions was explicitly requested. */
8824 if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_4_COMPATIBLE))
8826 else if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_5_COMPATIBLE))
8828 else if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_6_COMPATIBLE))
8830 else if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_8_COMPATIBLE))
8833 ffd->format = format;
8835 /* Override the default linear layout if this is a new-enough format. */
8836 if (format >= SVN_FS_FS__MIN_LAYOUT_FORMAT_OPTION_FORMAT)
8837 ffd->max_files_per_dir = SVN_FS_FS_DEFAULT_MAX_FILES_PER_DIR;
8839 /* Create the revision data directories. */
8840 if (ffd->max_files_per_dir)
8841 SVN_ERR(svn_io_make_dir_recursively(path_rev_shard(fs, 0, pool), pool));
8843 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path, PATH_REVS_DIR,
8847 /* Create the revprops directory. */
8848 if (ffd->max_files_per_dir)
8849 SVN_ERR(svn_io_make_dir_recursively(path_revprops_shard(fs, 0, pool),
8852 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path,
8857 /* Create the transaction directory. */
8858 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path, PATH_TXNS_DIR,
8862 /* Create the protorevs directory. */
8863 if (format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT)
8864 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path, PATH_TXN_PROTOS_DIR,
8868 /* Create the 'current' file. */
8869 SVN_ERR(svn_io_file_create(svn_fs_fs__path_current(fs, pool),
8870 (format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT
8871 ? "0\n" : "0 1 1\n"),
8873 SVN_ERR(svn_io_file_create(path_lock(fs, pool), "", pool));
8874 SVN_ERR(svn_fs_fs__set_uuid(fs, NULL, pool));
8876 SVN_ERR(write_revision_zero(fs));
8878 SVN_ERR(write_config(fs, pool));
8880 SVN_ERR(read_config(ffd, fs->path, pool));
8882 /* Create the min unpacked rev file. */
8883 if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
8884 SVN_ERR(svn_io_file_create(path_min_unpacked_rev(fs, pool), "0\n", pool));
8886 /* Create the txn-current file if the repository supports
8887 the transaction sequence file. */
8888 if (format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT)
8890 SVN_ERR(svn_io_file_create(path_txn_current(fs, pool),
8892 SVN_ERR(svn_io_file_create(path_txn_current_lock(fs, pool),
8896 /* This filesystem is ready. Stamp it with a format number. */
8897 SVN_ERR(write_format(path_format(fs, pool),
8898 ffd->format, ffd->max_files_per_dir, FALSE, pool));
8900 ffd->youngest_rev_cache = 0;
8901 return SVN_NO_ERROR;
8904 /* Part of the recovery procedure. Return the largest revision *REV in
8905 filesystem FS. Use POOL for temporary allocation. */
8906 static svn_error_t *
8907 recover_get_largest_revision(svn_fs_t *fs, svn_revnum_t *rev, apr_pool_t *pool)
8909 /* Discovering the largest revision in the filesystem would be an
8910 expensive operation if we did a readdir() or searched linearly,
8911 so we'll do a form of binary search. left is a revision that we
8912 know exists, right a revision that we know does not exist. */
8913 apr_pool_t *iterpool;
8914 svn_revnum_t left, right = 1;
8916 iterpool = svn_pool_create(pool);
8917 /* Keep doubling right, until we find a revision that doesn't exist. */
8923 err = open_pack_or_rev_file(&file, fs, right, iterpool);
8924 svn_pool_clear(iterpool);
8926 if (err && err->apr_err == SVN_ERR_FS_NO_SUCH_REVISION)
8928 svn_error_clear(err);
8939 /* We know that left exists and right doesn't. Do a normal bsearch to find
8940 the last revision. */
8941 while (left + 1 < right)
8943 svn_revnum_t probe = left + ((right - left) / 2);
8947 err = open_pack_or_rev_file(&file, fs, probe, iterpool);
8948 svn_pool_clear(iterpool);
8950 if (err && err->apr_err == SVN_ERR_FS_NO_SUCH_REVISION)
8952 svn_error_clear(err);
8962 svn_pool_destroy(iterpool);
8964 /* left is now the largest revision that exists. */
8966 return SVN_NO_ERROR;
8969 /* A baton for reading a fixed amount from an open file. For
8970 recover_find_max_ids() below. */
8971 struct recover_read_from_file_baton
8975 apr_off_t remaining;
8978 /* A stream read handler used by recover_find_max_ids() below.
8979 Read and return at most BATON->REMAINING bytes from the stream,
8980 returning nothing after that to indicate EOF. */
8981 static svn_error_t *
8982 read_handler_recover(void *baton, char *buffer, apr_size_t *len)
8984 struct recover_read_from_file_baton *b = baton;
8985 svn_filesize_t bytes_to_read = *len;
8987 if (b->remaining == 0)
8989 /* Return a successful read of zero bytes to signal EOF. */
8991 return SVN_NO_ERROR;
8994 if (bytes_to_read > b->remaining)
8995 bytes_to_read = b->remaining;
8996 b->remaining -= bytes_to_read;
8998 return svn_io_file_read_full2(b->file, buffer, (apr_size_t) bytes_to_read,
8999 len, NULL, b->pool);
9002 /* Part of the recovery procedure. Read the directory noderev at offset
9003 OFFSET of file REV_FILE (the revision file of revision REV of
9004 filesystem FS), and set MAX_NODE_ID and MAX_COPY_ID to be the node-id
9005 and copy-id of that node, if greater than the current value stored
9006 in either. Recurse into any child directories that were modified in
9009 MAX_NODE_ID and MAX_COPY_ID must be arrays of at least MAX_KEY_SIZE.
9011 Perform temporary allocation in POOL. */
9012 static svn_error_t *
9013 recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev,
9014 apr_file_t *rev_file, apr_off_t offset,
9015 char *max_node_id, char *max_copy_id,
9018 apr_hash_t *headers;
9020 representation_t *data_rep;
9021 struct rep_args *ra;
9022 struct recover_read_from_file_baton baton;
9023 svn_stream_t *stream;
9024 apr_hash_t *entries;
9025 apr_hash_index_t *hi;
9026 apr_pool_t *iterpool;
9028 SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool));
9029 SVN_ERR(read_header_block(&headers, svn_stream_from_aprfile2(rev_file, TRUE,
9033 /* Check that this is a directory. It should be. */
9034 value = svn_hash_gets(headers, HEADER_TYPE);
9035 if (value == NULL || strcmp(value, KIND_DIR) != 0)
9036 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
9037 _("Recovery encountered a non-directory node"));
9039 /* Get the data location. No data location indicates an empty directory. */
9040 value = svn_hash_gets(headers, HEADER_TEXT);
9042 return SVN_NO_ERROR;
9043 SVN_ERR(read_rep_offsets(&data_rep, value, NULL, FALSE, pool));
9045 /* If the directory's data representation wasn't changed in this revision,
9046 we've already scanned the directory's contents for noderevs, so we don't
9047 need to again. This will occur if a property is changed on a directory
9048 without changing the directory's contents. */
9049 if (data_rep->revision != rev)
9050 return SVN_NO_ERROR;
9052 /* We could use get_dir_contents(), but this is much cheaper. It does
9053 rely on directory entries being stored as PLAIN reps, though. */
9054 offset = data_rep->offset;
9055 SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool));
9056 SVN_ERR(read_rep_line(&ra, rev_file, pool));
9058 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
9059 _("Recovery encountered a deltified directory "
9062 /* Now create a stream that's allowed to read only as much data as is
9063 stored in the representation. */
9064 baton.file = rev_file;
9066 baton.remaining = data_rep->expanded_size;
9067 stream = svn_stream_create(&baton, pool);
9068 svn_stream_set_read(stream, read_handler_recover);
9070 /* Now read the entries from that stream. */
9071 entries = apr_hash_make(pool);
9072 SVN_ERR(svn_hash_read2(entries, stream, SVN_HASH_TERMINATOR, pool));
9073 SVN_ERR(svn_stream_close(stream));
9075 /* Now check each of the entries in our directory to find new node and
9076 copy ids, and recurse into new subdirectories. */
9077 iterpool = svn_pool_create(pool);
9078 for (hi = apr_hash_first(pool, entries); hi; hi = apr_hash_next(hi))
9082 svn_node_kind_t kind;
9084 const char *node_id, *copy_id;
9085 apr_off_t child_dir_offset;
9086 const svn_string_t *path = svn__apr_hash_index_val(hi);
9088 svn_pool_clear(iterpool);
9090 str_val = apr_pstrdup(iterpool, path->data);
9092 str = svn_cstring_tokenize(" ", &str_val);
9094 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
9095 _("Directory entry corrupt"));
9097 if (strcmp(str, KIND_FILE) == 0)
9098 kind = svn_node_file;
9099 else if (strcmp(str, KIND_DIR) == 0)
9100 kind = svn_node_dir;
9103 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
9104 _("Directory entry corrupt"));
9107 str = svn_cstring_tokenize(" ", &str_val);
9109 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
9110 _("Directory entry corrupt"));
9112 id = svn_fs_fs__id_parse(str, strlen(str), iterpool);
9114 if (svn_fs_fs__id_rev(id) != rev)
9116 /* If the node wasn't modified in this revision, we've already
9117 checked the node and copy id. */
9121 node_id = svn_fs_fs__id_node_id(id);
9122 copy_id = svn_fs_fs__id_copy_id(id);
9124 if (svn_fs_fs__key_compare(node_id, max_node_id) > 0)
9126 SVN_ERR_ASSERT(strlen(node_id) < MAX_KEY_SIZE);
9127 apr_cpystrn(max_node_id, node_id, MAX_KEY_SIZE);
9129 if (svn_fs_fs__key_compare(copy_id, max_copy_id) > 0)
9131 SVN_ERR_ASSERT(strlen(copy_id) < MAX_KEY_SIZE);
9132 apr_cpystrn(max_copy_id, copy_id, MAX_KEY_SIZE);
9135 if (kind == svn_node_file)
9138 child_dir_offset = svn_fs_fs__id_offset(id);
9139 SVN_ERR(recover_find_max_ids(fs, rev, rev_file, child_dir_offset,
9140 max_node_id, max_copy_id, iterpool));
9142 svn_pool_destroy(iterpool);
9144 return SVN_NO_ERROR;
9147 /* Return TRUE, if for REVISION in FS, we can find the revprop pack file.
9148 * Use POOL for temporary allocations.
9149 * Set *MISSING, if the reason is a missing manifest or pack file.
9151 static svn_boolean_t
9152 packed_revprop_available(svn_boolean_t *missing,
9154 svn_revnum_t revision,
9157 fs_fs_data_t *ffd = fs->fsap_data;
9158 svn_stringbuf_t *content = NULL;
9160 /* try to read the manifest file */
9161 const char *folder = path_revprops_pack_shard(fs, revision, pool);
9162 const char *manifest_path = svn_dirent_join(folder, PATH_MANIFEST, pool);
9164 svn_error_t *err = try_stringbuf_from_file(&content,
9170 /* if the manifest cannot be read, consider the pack files inaccessible
9171 * even if the file itself exists. */
9174 svn_error_clear(err);
9181 /* parse manifest content until we find the entry for REVISION.
9182 * Revision 0 is never packed. */
9183 revision = revision < ffd->max_files_per_dir
9185 : revision % ffd->max_files_per_dir;
9186 while (content->data)
9188 char *next = strchr(content->data, '\n');
9195 if (revision-- == 0)
9197 /* the respective pack file must exist (and be a file) */
9198 svn_node_kind_t kind;
9199 err = svn_io_check_path(svn_dirent_join(folder, content->data,
9204 svn_error_clear(err);
9208 *missing = kind == svn_node_none;
9209 return kind == svn_node_file;
9212 content->data = next;
9218 /* Baton used for recover_body below. */
9219 struct recover_baton {
9221 svn_cancel_func_t cancel_func;
9225 /* The work-horse for svn_fs_fs__recover, called with the FS
9226 write lock. This implements the svn_fs_fs__with_write_lock()
9227 'body' callback type. BATON is a 'struct recover_baton *'. */
9228 static svn_error_t *
9229 recover_body(void *baton, apr_pool_t *pool)
9231 struct recover_baton *b = baton;
9232 svn_fs_t *fs = b->fs;
9233 fs_fs_data_t *ffd = fs->fsap_data;
9234 svn_revnum_t max_rev;
9235 char next_node_id_buf[MAX_KEY_SIZE], next_copy_id_buf[MAX_KEY_SIZE];
9236 char *next_node_id = NULL, *next_copy_id = NULL;
9237 svn_revnum_t youngest_rev;
9238 svn_node_kind_t youngest_revprops_kind;
9240 /* Lose potentially corrupted data in temp files */
9241 SVN_ERR(cleanup_revprop_namespace(fs));
9243 /* We need to know the largest revision in the filesystem. */
9244 SVN_ERR(recover_get_largest_revision(fs, &max_rev, pool));
9246 /* Get the expected youngest revision */
9247 SVN_ERR(get_youngest(&youngest_rev, fs->path, pool));
9251 Since the revprops file is written after the revs file, the true
9252 maximum available revision is the youngest one for which both are
9253 present. That's probably the same as the max_rev we just found,
9254 but if it's not, we could, in theory, repeatedly decrement
9255 max_rev until we find a revision that has both a revs and
9256 revprops file, then write db/current with that.
9258 But we choose not to. If a repository is so corrupt that it's
9259 missing at least one revprops file, we shouldn't assume that the
9260 youngest revision for which both the revs and revprops files are
9261 present is healthy. In other words, we're willing to recover
9262 from a missing or out-of-date db/current file, because db/current
9263 is truly redundant -- it's basically a cache so we don't have to
9264 find max_rev each time, albeit a cache with unusual semantics,
9265 since it also officially defines when a revision goes live. But
9266 if we're missing more than the cache, it's time to back out and
9267 let the admin reconstruct things by hand: correctness at that
9268 point may depend on external things like checking a commit email
9269 list, looking in particular working copies, etc.
9271 This policy matches well with a typical naive backup scenario.
9272 Say you're rsyncing your FSFS repository nightly to the same
9273 location. Once revs and revprops are written, you've got the
9274 maximum rev; if the backup should bomb before db/current is
9275 written, then db/current could stay arbitrarily out-of-date, but
9276 we can still recover. It's a small window, but we might as well
9279 /* Even if db/current were missing, it would be created with 0 by
9280 get_youngest(), so this conditional remains valid. */
9281 if (youngest_rev > max_rev)
9282 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
9283 _("Expected current rev to be <= %ld "
9284 "but found %ld"), max_rev, youngest_rev);
9286 /* We only need to search for maximum IDs for old FS formats which
9287 se global ID counters. */
9288 if (ffd->format < SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT)
9290 /* Next we need to find the maximum node id and copy id in use across the
9291 filesystem. Unfortunately, the only way we can get this information
9292 is to scan all the noderevs of all the revisions and keep track as
9295 apr_pool_t *iterpool = svn_pool_create(pool);
9296 char max_node_id[MAX_KEY_SIZE] = "0", max_copy_id[MAX_KEY_SIZE] = "0";
9299 for (rev = 0; rev <= max_rev; rev++)
9301 apr_file_t *rev_file;
9302 apr_off_t root_offset;
9304 svn_pool_clear(iterpool);
9307 SVN_ERR(b->cancel_func(b->cancel_baton));
9309 SVN_ERR(open_pack_or_rev_file(&rev_file, fs, rev, iterpool));
9310 SVN_ERR(get_root_changes_offset(&root_offset, NULL, rev_file, fs, rev,
9312 SVN_ERR(recover_find_max_ids(fs, rev, rev_file, root_offset,
9313 max_node_id, max_copy_id, iterpool));
9314 SVN_ERR(svn_io_file_close(rev_file, iterpool));
9316 svn_pool_destroy(iterpool);
9318 /* Now that we finally have the maximum revision, node-id and copy-id, we
9319 can bump the two ids to get the next of each. */
9320 len = strlen(max_node_id);
9321 svn_fs_fs__next_key(max_node_id, &len, next_node_id_buf);
9322 next_node_id = next_node_id_buf;
9323 len = strlen(max_copy_id);
9324 svn_fs_fs__next_key(max_copy_id, &len, next_copy_id_buf);
9325 next_copy_id = next_copy_id_buf;
9328 /* Before setting current, verify that there is a revprops file
9329 for the youngest revision. (Issue #2992) */
9330 SVN_ERR(svn_io_check_path(path_revprops(fs, max_rev, pool),
9331 &youngest_revprops_kind, pool));
9332 if (youngest_revprops_kind == svn_node_none)
9334 svn_boolean_t missing = TRUE;
9335 if (!packed_revprop_available(&missing, fs, max_rev, pool))
9339 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
9340 _("Revision %ld has a revs file but no "
9346 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
9347 _("Revision %ld has a revs file but the "
9348 "revprops file is inaccessible"),
9353 else if (youngest_revprops_kind != svn_node_file)
9355 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
9356 _("Revision %ld has a non-file where its "
9357 "revprops file should be"),
9361 /* Prune younger-than-(newfound-youngest) revisions from the rep
9362 cache if sharing is enabled taking care not to create the cache
9363 if it does not exist. */
9364 if (ffd->rep_sharing_allowed)
9366 svn_boolean_t rep_cache_exists;
9368 SVN_ERR(svn_fs_fs__exists_rep_cache(&rep_cache_exists, fs, pool));
9369 if (rep_cache_exists)
9370 SVN_ERR(svn_fs_fs__del_rep_reference(fs, max_rev, pool));
9373 /* Now store the discovered youngest revision, and the next IDs if
9374 relevant, in a new 'current' file. */
9375 return write_current(fs, max_rev, next_node_id, next_copy_id, pool);
9378 /* This implements the fs_library_vtable_t.recover() API. */
9380 svn_fs_fs__recover(svn_fs_t *fs,
9381 svn_cancel_func_t cancel_func, void *cancel_baton,
9384 struct recover_baton b;
9386 /* We have no way to take out an exclusive lock in FSFS, so we're
9387 restricted as to the types of recovery we can do. Luckily,
9388 we just want to recreate the 'current' file, and we can do that just
9389 by blocking other writers. */
9391 b.cancel_func = cancel_func;
9392 b.cancel_baton = cancel_baton;
9393 return svn_fs_fs__with_write_lock(fs, recover_body, &b, pool);
9397 svn_fs_fs__set_uuid(svn_fs_t *fs,
9402 apr_size_t my_uuid_len;
9403 const char *tmp_path;
9404 const char *uuid_path = path_uuid(fs, pool);
9407 uuid = svn_uuid_generate(pool);
9409 /* Make sure we have a copy in FS->POOL, and append a newline. */
9410 my_uuid = apr_pstrcat(fs->pool, uuid, "\n", (char *)NULL);
9411 my_uuid_len = strlen(my_uuid);
9413 SVN_ERR(svn_io_write_unique(&tmp_path,
9414 svn_dirent_dirname(uuid_path, pool),
9415 my_uuid, my_uuid_len,
9416 svn_io_file_del_none, pool));
9418 /* We use the permissions of the 'current' file, because the 'uuid'
9419 file does not exist during repository creation. */
9420 SVN_ERR(move_into_place(tmp_path, uuid_path,
9421 svn_fs_fs__path_current(fs, pool), pool));
9423 /* Remove the newline we added, and stash the UUID. */
9424 my_uuid[my_uuid_len - 1] = '\0';
9427 return SVN_NO_ERROR;
9430 /** Node origin lazy cache. */
9432 /* If directory PATH does not exist, create it and give it the same
9433 permissions as FS_path.*/
9435 svn_fs_fs__ensure_dir_exists(const char *path,
9436 const char *fs_path,
9439 svn_error_t *err = svn_io_dir_make(path, APR_OS_DEFAULT, pool);
9440 if (err && APR_STATUS_IS_EEXIST(err->apr_err))
9442 svn_error_clear(err);
9443 return SVN_NO_ERROR;
9447 /* We successfully created a new directory. Dup the permissions
9449 return svn_io_copy_perms(fs_path, path, pool);
9452 /* Set *NODE_ORIGINS to a hash mapping 'const char *' node IDs to
9453 'svn_string_t *' node revision IDs. Use POOL for allocations. */
9454 static svn_error_t *
9455 get_node_origins_from_file(svn_fs_t *fs,
9456 apr_hash_t **node_origins,
9457 const char *node_origins_file,
9462 svn_stream_t *stream;
9464 *node_origins = NULL;
9465 err = svn_io_file_open(&fd, node_origins_file,
9466 APR_READ, APR_OS_DEFAULT, pool);
9467 if (err && APR_STATUS_IS_ENOENT(err->apr_err))
9469 svn_error_clear(err);
9470 return SVN_NO_ERROR;
9474 stream = svn_stream_from_aprfile2(fd, FALSE, pool);
9475 *node_origins = apr_hash_make(pool);
9476 SVN_ERR(svn_hash_read2(*node_origins, stream, SVN_HASH_TERMINATOR, pool));
9477 return svn_stream_close(stream);
9481 svn_fs_fs__get_node_origin(const svn_fs_id_t **origin_id,
9483 const char *node_id,
9486 apr_hash_t *node_origins;
9489 SVN_ERR(get_node_origins_from_file(fs, &node_origins,
9490 path_node_origin(fs, node_id, pool),
9494 svn_string_t *origin_id_str =
9495 svn_hash_gets(node_origins, node_id);
9497 *origin_id = svn_fs_fs__id_parse(origin_id_str->data,
9498 origin_id_str->len, pool);
9500 return SVN_NO_ERROR;
9504 /* Helper for svn_fs_fs__set_node_origin. Takes a NODE_ID/NODE_REV_ID
9505 pair and adds it to the NODE_ORIGINS_PATH file. */
9506 static svn_error_t *
9507 set_node_origins_for_file(svn_fs_t *fs,
9508 const char *node_origins_path,
9509 const char *node_id,
9510 svn_string_t *node_rev_id,
9513 const char *path_tmp;
9514 svn_stream_t *stream;
9515 apr_hash_t *origins_hash;
9516 svn_string_t *old_node_rev_id;
9518 SVN_ERR(svn_fs_fs__ensure_dir_exists(svn_dirent_join(fs->path,
9519 PATH_NODE_ORIGINS_DIR,
9523 /* Read the previously existing origins (if any), and merge our
9525 SVN_ERR(get_node_origins_from_file(fs, &origins_hash,
9526 node_origins_path, pool));
9528 origins_hash = apr_hash_make(pool);
9530 old_node_rev_id = svn_hash_gets(origins_hash, node_id);
9532 if (old_node_rev_id && !svn_string_compare(node_rev_id, old_node_rev_id))
9533 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
9534 _("Node origin for '%s' exists with a different "
9535 "value (%s) than what we were about to store "
9537 node_id, old_node_rev_id->data, node_rev_id->data);
9539 svn_hash_sets(origins_hash, node_id, node_rev_id);
9541 /* Sure, there's a race condition here. Two processes could be
9542 trying to add different cache elements to the same file at the
9543 same time, and the entries added by the first one to write will
9544 be lost. But this is just a cache of reconstructible data, so
9545 we'll accept this problem in return for not having to deal with
9546 locking overhead. */
9548 /* Create a temporary file, write out our hash, and close the file. */
9549 SVN_ERR(svn_stream_open_unique(&stream, &path_tmp,
9550 svn_dirent_dirname(node_origins_path, pool),
9551 svn_io_file_del_none, pool, pool));
9552 SVN_ERR(svn_hash_write2(origins_hash, stream, SVN_HASH_TERMINATOR, pool));
9553 SVN_ERR(svn_stream_close(stream));
9555 /* Rename the temp file as the real destination */
9556 return svn_io_file_rename(path_tmp, node_origins_path, pool);
9561 svn_fs_fs__set_node_origin(svn_fs_t *fs,
9562 const char *node_id,
9563 const svn_fs_id_t *node_rev_id,
9567 const char *filename = path_node_origin(fs, node_id, pool);
9569 err = set_node_origins_for_file(fs, filename,
9571 svn_fs_fs__id_unparse(node_rev_id, pool),
9573 if (err && APR_STATUS_IS_EACCES(err->apr_err))
9575 /* It's just a cache; stop trying if I can't write. */
9576 svn_error_clear(err);
9579 return svn_error_trace(err);
9584 svn_fs_fs__list_transactions(apr_array_header_t **names_p,
9588 const char *txn_dir;
9589 apr_hash_t *dirents;
9590 apr_hash_index_t *hi;
9591 apr_array_header_t *names;
9592 apr_size_t ext_len = strlen(PATH_EXT_TXN);
9594 names = apr_array_make(pool, 1, sizeof(const char *));
9596 /* Get the transactions directory. */
9597 txn_dir = svn_dirent_join(fs->path, PATH_TXNS_DIR, pool);
9599 /* Now find a listing of this directory. */
9600 SVN_ERR(svn_io_get_dirents3(&dirents, txn_dir, TRUE, pool, pool));
9602 /* Loop through all the entries and return anything that ends with '.txn'. */
9603 for (hi = apr_hash_first(pool, dirents); hi; hi = apr_hash_next(hi))
9605 const char *name = svn__apr_hash_index_key(hi);
9606 apr_ssize_t klen = svn__apr_hash_index_klen(hi);
9609 /* The name must end with ".txn" to be considered a transaction. */
9610 if ((apr_size_t) klen <= ext_len
9611 || (strcmp(name + klen - ext_len, PATH_EXT_TXN)) != 0)
9614 /* Truncate the ".txn" extension and store the ID. */
9615 id = apr_pstrndup(pool, name, strlen(name) - ext_len);
9616 APR_ARRAY_PUSH(names, const char *) = id;
9621 return SVN_NO_ERROR;
9625 svn_fs_fs__open_txn(svn_fs_txn_t **txn_p,
9631 svn_node_kind_t kind;
9632 transaction_t *local_txn;
9634 /* First check to see if the directory exists. */
9635 SVN_ERR(svn_io_check_path(path_txn_dir(fs, name, pool), &kind, pool));
9637 /* Did we find it? */
9638 if (kind != svn_node_dir)
9639 return svn_error_createf(SVN_ERR_FS_NO_SUCH_TRANSACTION, NULL,
9640 _("No such transaction '%s'"),
9643 txn = apr_pcalloc(pool, sizeof(*txn));
9645 /* Read in the root node of this transaction. */
9646 txn->id = apr_pstrdup(pool, name);
9649 SVN_ERR(svn_fs_fs__get_txn(&local_txn, fs, name, pool));
9651 txn->base_rev = svn_fs_fs__id_rev(local_txn->base_id);
9653 txn->vtable = &txn_vtable;
9656 return SVN_NO_ERROR;
9660 svn_fs_fs__txn_proplist(apr_hash_t **table_p,
9664 apr_hash_t *proplist = apr_hash_make(pool);
9665 SVN_ERR(get_txn_proplist(proplist, txn->fs, txn->id, pool));
9666 *table_p = proplist;
9668 return SVN_NO_ERROR;
9672 svn_fs_fs__delete_node_revision(svn_fs_t *fs,
9673 const svn_fs_id_t *id,
9676 node_revision_t *noderev;
9678 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, id, pool));
9680 /* Delete any mutable property representation. */
9681 if (noderev->prop_rep && noderev->prop_rep->txn_id)
9682 SVN_ERR(svn_io_remove_file2(path_txn_node_props(fs, id, pool), FALSE,
9685 /* Delete any mutable data representation. */
9686 if (noderev->data_rep && noderev->data_rep->txn_id
9687 && noderev->kind == svn_node_dir)
9689 fs_fs_data_t *ffd = fs->fsap_data;
9690 SVN_ERR(svn_io_remove_file2(path_txn_node_children(fs, id, pool), FALSE,
9693 /* remove the corresponding entry from the cache, if such exists */
9694 if (ffd->txn_dir_cache)
9696 const char *key = svn_fs_fs__id_unparse(id, pool)->data;
9697 SVN_ERR(svn_cache__set(ffd->txn_dir_cache, key, NULL, pool));
9701 return svn_io_remove_file2(path_txn_node_rev(fs, id, pool), FALSE, pool);
9709 svn_fs_fs__revision_prop(svn_string_t **value_p,
9712 const char *propname,
9717 SVN_ERR(svn_fs__check_fs(fs, TRUE));
9718 SVN_ERR(svn_fs_fs__revision_proplist(&table, fs, rev, pool));
9720 *value_p = svn_hash_gets(table, propname);
9722 return SVN_NO_ERROR;
9726 /* Baton used for change_rev_prop_body below. */
9727 struct change_rev_prop_baton {
9731 const svn_string_t *const *old_value_p;
9732 const svn_string_t *value;
9735 /* The work-horse for svn_fs_fs__change_rev_prop, called with the FS
9736 write lock. This implements the svn_fs_fs__with_write_lock()
9737 'body' callback type. BATON is a 'struct change_rev_prop_baton *'. */
9738 static svn_error_t *
9739 change_rev_prop_body(void *baton, apr_pool_t *pool)
9741 struct change_rev_prop_baton *cb = baton;
9744 SVN_ERR(svn_fs_fs__revision_proplist(&table, cb->fs, cb->rev, pool));
9746 if (cb->old_value_p)
9748 const svn_string_t *wanted_value = *cb->old_value_p;
9749 const svn_string_t *present_value = svn_hash_gets(table, cb->name);
9750 if ((!wanted_value != !present_value)
9751 || (wanted_value && present_value
9752 && !svn_string_compare(wanted_value, present_value)))
9754 /* What we expected isn't what we found. */
9755 return svn_error_createf(SVN_ERR_FS_PROP_BASEVALUE_MISMATCH, NULL,
9756 _("revprop '%s' has unexpected value in "
9762 svn_hash_sets(table, cb->name, cb->value);
9764 return set_revision_proplist(cb->fs, cb->rev, table, pool);
9768 svn_fs_fs__change_rev_prop(svn_fs_t *fs,
9771 const svn_string_t *const *old_value_p,
9772 const svn_string_t *value,
9775 struct change_rev_prop_baton cb;
9777 SVN_ERR(svn_fs__check_fs(fs, TRUE));
9782 cb.old_value_p = old_value_p;
9785 return svn_fs_fs__with_write_lock(fs, change_rev_prop_body, &cb, pool);
9790 /*** Transactions ***/
9793 svn_fs_fs__get_txn_ids(const svn_fs_id_t **root_id_p,
9794 const svn_fs_id_t **base_root_id_p,
9796 const char *txn_name,
9800 SVN_ERR(svn_fs_fs__get_txn(&txn, fs, txn_name, pool));
9801 *root_id_p = txn->root_id;
9802 *base_root_id_p = txn->base_id;
9803 return SVN_NO_ERROR;
9807 /* Generic transaction operations. */
9810 svn_fs_fs__txn_prop(svn_string_t **value_p,
9812 const char *propname,
9816 svn_fs_t *fs = txn->fs;
9818 SVN_ERR(svn_fs__check_fs(fs, TRUE));
9819 SVN_ERR(svn_fs_fs__txn_proplist(&table, txn, pool));
9821 *value_p = svn_hash_gets(table, propname);
9823 return SVN_NO_ERROR;
9827 svn_fs_fs__begin_txn(svn_fs_txn_t **txn_p,
9835 apr_array_header_t *props = apr_array_make(pool, 3, sizeof(svn_prop_t));
9837 SVN_ERR(svn_fs__check_fs(fs, TRUE));
9839 SVN_ERR(svn_fs_fs__create_txn(txn_p, fs, rev, pool));
9841 /* Put a datestamp on the newly created txn, so we always know
9842 exactly how old it is. (This will help sysadmins identify
9843 long-abandoned txns that may need to be manually removed.) When
9844 a txn is promoted to a revision, this property will be
9845 automatically overwritten with a revision datestamp. */
9846 date.data = svn_time_to_cstring(apr_time_now(), pool);
9847 date.len = strlen(date.data);
9849 prop.name = SVN_PROP_REVISION_DATE;
9851 APR_ARRAY_PUSH(props, svn_prop_t) = prop;
9853 /* Set temporary txn props that represent the requested 'flags'
9855 if (flags & SVN_FS_TXN_CHECK_OOD)
9857 prop.name = SVN_FS__PROP_TXN_CHECK_OOD;
9858 prop.value = svn_string_create("true", pool);
9859 APR_ARRAY_PUSH(props, svn_prop_t) = prop;
9862 if (flags & SVN_FS_TXN_CHECK_LOCKS)
9864 prop.name = SVN_FS__PROP_TXN_CHECK_LOCKS;
9865 prop.value = svn_string_create("true", pool);
9866 APR_ARRAY_PUSH(props, svn_prop_t) = prop;
9869 return svn_fs_fs__change_txn_props(*txn_p, props, pool);
9873 /****** Packing FSFS shards *********/
9875 /* Write a file FILENAME in directory FS_PATH, containing a single line
9876 * with the number REVNUM in ASCII decimal. Move the file into place
9877 * atomically, overwriting any existing file.
9879 * Similar to write_current(). */
9880 static svn_error_t *
9881 write_revnum_file(const char *fs_path,
9882 const char *filename,
9883 svn_revnum_t revnum,
9884 apr_pool_t *scratch_pool)
9886 const char *final_path, *tmp_path;
9887 svn_stream_t *tmp_stream;
9889 final_path = svn_dirent_join(fs_path, filename, scratch_pool);
9890 SVN_ERR(svn_stream_open_unique(&tmp_stream, &tmp_path, fs_path,
9891 svn_io_file_del_none,
9892 scratch_pool, scratch_pool));
9893 SVN_ERR(svn_stream_printf(tmp_stream, scratch_pool, "%ld\n", revnum));
9894 SVN_ERR(svn_stream_close(tmp_stream));
9895 SVN_ERR(move_into_place(tmp_path, final_path, final_path, scratch_pool));
9896 return SVN_NO_ERROR;
9899 /* Pack the revision SHARD containing exactly MAX_FILES_PER_DIR revisions
9900 * from SHARD_PATH into the PACK_FILE_DIR, using POOL for allocations.
9901 * CANCEL_FUNC and CANCEL_BATON are what you think they are.
9903 * If for some reason we detect a partial packing already performed, we
9904 * remove the pack file and start again.
9906 static svn_error_t *
9907 pack_rev_shard(const char *pack_file_dir,
9908 const char *shard_path,
9910 int max_files_per_dir,
9911 svn_cancel_func_t cancel_func,
9915 const char *pack_file_path, *manifest_file_path;
9916 svn_stream_t *pack_stream, *manifest_stream;
9917 svn_revnum_t start_rev, end_rev, rev;
9918 apr_off_t next_offset;
9919 apr_pool_t *iterpool;
9921 /* Some useful paths. */
9922 pack_file_path = svn_dirent_join(pack_file_dir, PATH_PACKED, pool);
9923 manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, pool);
9925 /* Remove any existing pack file for this shard, since it is incomplete. */
9926 SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton,
9929 /* Create the new directory and pack and manifest files. */
9930 SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, pool));
9931 SVN_ERR(svn_stream_open_writable(&pack_stream, pack_file_path, pool,
9933 SVN_ERR(svn_stream_open_writable(&manifest_stream, manifest_file_path,
9936 start_rev = (svn_revnum_t) (shard * max_files_per_dir);
9937 end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1);
9939 iterpool = svn_pool_create(pool);
9941 /* Iterate over the revisions in this shard, squashing them together. */
9942 for (rev = start_rev; rev <= end_rev; rev++)
9944 svn_stream_t *rev_stream;
9948 svn_pool_clear(iterpool);
9950 /* Get the size of the file. */
9951 path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
9953 SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool));
9955 /* Update the manifest. */
9956 SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%" APR_OFF_T_FMT
9957 "\n", next_offset));
9958 next_offset += finfo.size;
9960 /* Copy all the bits from the rev file to the end of the pack file. */
9961 SVN_ERR(svn_stream_open_readonly(&rev_stream, path, iterpool, iterpool));
9962 SVN_ERR(svn_stream_copy3(rev_stream, svn_stream_disown(pack_stream,
9964 cancel_func, cancel_baton, iterpool));
9967 SVN_ERR(svn_stream_close(manifest_stream));
9968 SVN_ERR(svn_stream_close(pack_stream));
9969 SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool));
9970 SVN_ERR(svn_io_set_file_read_only(pack_file_path, FALSE, iterpool));
9971 SVN_ERR(svn_io_set_file_read_only(manifest_file_path, FALSE, iterpool));
9973 svn_pool_destroy(iterpool);
9975 return SVN_NO_ERROR;
9978 /* Copy revprop files for revisions [START_REV, END_REV) from SHARD_PATH
9979 * to the pack file at PACK_FILE_NAME in PACK_FILE_DIR.
9981 * The file sizes have already been determined and written to SIZES.
9982 * Please note that this function will be executed while the filesystem
9983 * has been locked and that revprops files will therefore not be modified
9984 * while the pack is in progress.
9986 * COMPRESSION_LEVEL defines how well the resulting pack file shall be
9987 * compressed or whether is shall be compressed at all. TOTAL_SIZE is
9988 * a hint on which initial buffer size we should use to hold the pack file
9991 * CANCEL_FUNC and CANCEL_BATON are used as usual. Temporary allocations
9992 * are done in SCRATCH_POOL.
9994 static svn_error_t *
9995 copy_revprops(const char *pack_file_dir,
9996 const char *pack_filename,
9997 const char *shard_path,
9998 svn_revnum_t start_rev,
9999 svn_revnum_t end_rev,
10000 apr_array_header_t *sizes,
10001 apr_size_t total_size,
10002 int compression_level,
10003 svn_cancel_func_t cancel_func,
10004 void *cancel_baton,
10005 apr_pool_t *scratch_pool)
10007 svn_stream_t *pack_stream;
10008 apr_file_t *pack_file;
10010 apr_pool_t *iterpool = svn_pool_create(scratch_pool);
10011 svn_stream_t *stream;
10013 /* create empty data buffer and a write stream on top of it */
10014 svn_stringbuf_t *uncompressed
10015 = svn_stringbuf_create_ensure(total_size, scratch_pool);
10016 svn_stringbuf_t *compressed
10017 = svn_stringbuf_create_empty(scratch_pool);
10018 pack_stream = svn_stream_from_stringbuf(uncompressed, scratch_pool);
10020 /* write the pack file header */
10021 SVN_ERR(serialize_revprops_header(pack_stream, start_rev, sizes, 0,
10022 sizes->nelts, iterpool));
10024 /* Some useful paths. */
10025 SVN_ERR(svn_io_file_open(&pack_file, svn_dirent_join(pack_file_dir,
10028 APR_WRITE | APR_CREATE, APR_OS_DEFAULT,
10031 /* Iterate over the revisions in this shard, squashing them together. */
10032 for (rev = start_rev; rev <= end_rev; rev++)
10036 svn_pool_clear(iterpool);
10038 /* Construct the file name. */
10039 path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
10042 /* Copy all the bits from the non-packed revprop file to the end of
10043 * the pack file. */
10044 SVN_ERR(svn_stream_open_readonly(&stream, path, iterpool, iterpool));
10045 SVN_ERR(svn_stream_copy3(stream, pack_stream,
10046 cancel_func, cancel_baton, iterpool));
10049 /* flush stream buffers to content buffer */
10050 SVN_ERR(svn_stream_close(pack_stream));
10052 /* compress the content (or just store it for COMPRESSION_LEVEL 0) */
10053 SVN_ERR(svn__compress(svn_stringbuf__morph_into_string(uncompressed),
10054 compressed, compression_level));
10056 /* write the pack file content to disk */
10057 stream = svn_stream_from_aprfile2(pack_file, FALSE, scratch_pool);
10058 SVN_ERR(svn_stream_write(stream, compressed->data, &compressed->len));
10059 SVN_ERR(svn_stream_close(stream));
10061 svn_pool_destroy(iterpool);
10063 return SVN_NO_ERROR;
10066 /* For the revprop SHARD at SHARD_PATH with exactly MAX_FILES_PER_DIR
10067 * revprop files in it, create a packed shared at PACK_FILE_DIR.
10069 * COMPRESSION_LEVEL defines how well the resulting pack file shall be
10070 * compressed or whether is shall be compressed at all. Individual pack
10071 * file containing more than one revision will be limited to a size of
10072 * MAX_PACK_SIZE bytes before compression.
10074 * CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary
10075 * allocations are done in SCRATCH_POOL.
10077 static svn_error_t *
10078 pack_revprops_shard(const char *pack_file_dir,
10079 const char *shard_path,
10081 int max_files_per_dir,
10082 apr_off_t max_pack_size,
10083 int compression_level,
10084 svn_cancel_func_t cancel_func,
10085 void *cancel_baton,
10086 apr_pool_t *scratch_pool)
10088 const char *manifest_file_path, *pack_filename = NULL;
10089 svn_stream_t *manifest_stream;
10090 svn_revnum_t start_rev, end_rev, rev;
10091 apr_off_t total_size;
10092 apr_pool_t *iterpool = svn_pool_create(scratch_pool);
10093 apr_array_header_t *sizes;
10095 /* Some useful paths. */
10096 manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST,
10099 /* Remove any existing pack file for this shard, since it is incomplete. */
10100 SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton,
10103 /* Create the new directory and manifest file stream. */
10104 SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool));
10105 SVN_ERR(svn_stream_open_writable(&manifest_stream, manifest_file_path,
10106 scratch_pool, scratch_pool));
10108 /* revisions to handle. Special case: revision 0 */
10109 start_rev = (svn_revnum_t) (shard * max_files_per_dir);
10110 end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1);
10111 if (start_rev == 0)
10114 /* initialize the revprop size info */
10115 sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_off_t));
10116 total_size = 2 * SVN_INT64_BUFFER_SIZE;
10118 /* Iterate over the revisions in this shard, determine their size and
10119 * squashing them together into pack files. */
10120 for (rev = start_rev; rev <= end_rev; rev++)
10125 svn_pool_clear(iterpool);
10127 /* Get the size of the file. */
10128 path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
10130 SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool));
10132 /* if we already have started a pack file and this revprop cannot be
10133 * appended to it, write the previous pack file. */
10134 if (sizes->nelts != 0 &&
10135 total_size + SVN_INT64_BUFFER_SIZE + finfo.size > max_pack_size)
10137 SVN_ERR(copy_revprops(pack_file_dir, pack_filename, shard_path,
10138 start_rev, rev-1, sizes, (apr_size_t)total_size,
10139 compression_level, cancel_func, cancel_baton,
10142 /* next pack file starts empty again */
10143 apr_array_clear(sizes);
10144 total_size = 2 * SVN_INT64_BUFFER_SIZE;
10148 /* Update the manifest. Allocate a file name for the current pack
10149 * file if it is a new one */
10150 if (sizes->nelts == 0)
10151 pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev);
10153 SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%s\n",
10156 /* add to list of files to put into the current pack file */
10157 APR_ARRAY_PUSH(sizes, apr_off_t) = finfo.size;
10158 total_size += SVN_INT64_BUFFER_SIZE + finfo.size;
10161 /* write the last pack file */
10162 if (sizes->nelts != 0)
10163 SVN_ERR(copy_revprops(pack_file_dir, pack_filename, shard_path,
10164 start_rev, rev-1, sizes, (apr_size_t)total_size,
10165 compression_level, cancel_func, cancel_baton,
10168 /* flush the manifest file and update permissions */
10169 SVN_ERR(svn_stream_close(manifest_stream));
10170 SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool));
10172 svn_pool_destroy(iterpool);
10174 return SVN_NO_ERROR;
10177 /* Delete the non-packed revprop SHARD at SHARD_PATH with exactly
10178 * MAX_FILES_PER_DIR revprop files in it. If this is shard 0, keep the
10179 * revprop file for revision 0.
10181 * CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary
10182 * allocations are done in SCRATCH_POOL.
10184 static svn_error_t *
10185 delete_revprops_shard(const char *shard_path,
10187 int max_files_per_dir,
10188 svn_cancel_func_t cancel_func,
10189 void *cancel_baton,
10190 apr_pool_t *scratch_pool)
10194 apr_pool_t *iterpool = svn_pool_create(scratch_pool);
10197 /* delete all files except the one for revision 0 */
10198 for (i = 1; i < max_files_per_dir; ++i)
10200 const char *path = svn_dirent_join(shard_path,
10201 apr_psprintf(iterpool, "%d", i),
10204 SVN_ERR((*cancel_func)(cancel_baton));
10206 SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
10207 svn_pool_clear(iterpool);
10210 svn_pool_destroy(iterpool);
10213 SVN_ERR(svn_io_remove_dir2(shard_path, TRUE,
10214 cancel_func, cancel_baton, scratch_pool));
10216 return SVN_NO_ERROR;
10219 /* In the file system at FS_PATH, pack the SHARD in REVS_DIR and
10220 * REVPROPS_DIR containing exactly MAX_FILES_PER_DIR revisions, using POOL
10221 * for allocations. REVPROPS_DIR will be NULL if revprop packing is not
10222 * supported. COMPRESSION_LEVEL and MAX_PACK_SIZE will be ignored in that
10225 * CANCEL_FUNC and CANCEL_BATON are what you think they are; similarly
10226 * NOTIFY_FUNC and NOTIFY_BATON.
10228 * If for some reason we detect a partial packing already performed, we
10229 * remove the pack file and start again.
10231 static svn_error_t *
10232 pack_shard(const char *revs_dir,
10233 const char *revsprops_dir,
10234 const char *fs_path,
10236 int max_files_per_dir,
10237 apr_off_t max_pack_size,
10238 int compression_level,
10239 svn_fs_pack_notify_t notify_func,
10240 void *notify_baton,
10241 svn_cancel_func_t cancel_func,
10242 void *cancel_baton,
10245 const char *rev_shard_path, *rev_pack_file_dir;
10246 const char *revprops_shard_path, *revprops_pack_file_dir;
10248 /* Notify caller we're starting to pack this shard. */
10250 SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start,
10253 /* Some useful paths. */
10254 rev_pack_file_dir = svn_dirent_join(revs_dir,
10256 "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
10259 rev_shard_path = svn_dirent_join(revs_dir,
10260 apr_psprintf(pool, "%" APR_INT64_T_FMT, shard),
10263 /* pack the revision content */
10264 SVN_ERR(pack_rev_shard(rev_pack_file_dir, rev_shard_path,
10265 shard, max_files_per_dir,
10266 cancel_func, cancel_baton, pool));
10268 /* if enabled, pack the revprops in an equivalent way */
10271 revprops_pack_file_dir = svn_dirent_join(revsprops_dir,
10273 "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
10276 revprops_shard_path = svn_dirent_join(revsprops_dir,
10277 apr_psprintf(pool, "%" APR_INT64_T_FMT, shard),
10280 SVN_ERR(pack_revprops_shard(revprops_pack_file_dir, revprops_shard_path,
10281 shard, max_files_per_dir,
10282 (int)(0.9 * max_pack_size),
10284 cancel_func, cancel_baton, pool));
10287 /* Update the min-unpacked-rev file to reflect our newly packed shard.
10288 * (This doesn't update ffd->min_unpacked_rev. That will be updated by
10289 * update_min_unpacked_rev() when necessary.) */
10290 SVN_ERR(write_revnum_file(fs_path, PATH_MIN_UNPACKED_REV,
10291 (svn_revnum_t)((shard + 1) * max_files_per_dir),
10294 /* Finally, remove the existing shard directories. */
10295 SVN_ERR(svn_io_remove_dir2(rev_shard_path, TRUE,
10296 cancel_func, cancel_baton, pool));
10298 SVN_ERR(delete_revprops_shard(revprops_shard_path,
10299 shard, max_files_per_dir,
10300 cancel_func, cancel_baton, pool));
10302 /* Notify caller we're starting to pack this shard. */
10304 SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_end,
10307 return SVN_NO_ERROR;
10313 svn_fs_pack_notify_t notify_func;
10314 void *notify_baton;
10315 svn_cancel_func_t cancel_func;
10316 void *cancel_baton;
10320 /* The work-horse for svn_fs_fs__pack, called with the FS write lock.
10321 This implements the svn_fs_fs__with_write_lock() 'body' callback
10322 type. BATON is a 'struct pack_baton *'.
10324 WARNING: if you add a call to this function, please note:
10325 The code currently assumes that any piece of code running with
10326 the write-lock set can rely on the ffd->min_unpacked_rev and
10327 ffd->min_unpacked_revprop caches to be up-to-date (and, by
10328 extension, on not having to use a retry when calling
10329 svn_fs_fs__path_rev_absolute() and friends). If you add a call
10330 to this function, consider whether you have to call
10331 update_min_unpacked_rev().
10332 See this thread: http://thread.gmane.org/1291206765.3782.3309.camel@edith
10334 static svn_error_t *
10335 pack_body(void *baton,
10338 struct pack_baton *pb = baton;
10339 fs_fs_data_t ffd = {0};
10340 apr_int64_t completed_shards;
10342 svn_revnum_t youngest;
10343 apr_pool_t *iterpool;
10344 const char *rev_data_path;
10345 const char *revprops_data_path = NULL;
10347 /* read repository settings */
10348 SVN_ERR(read_format(&ffd.format, &ffd.max_files_per_dir,
10349 path_format(pb->fs, pool), pool));
10350 SVN_ERR(check_format(ffd.format));
10351 SVN_ERR(read_config(&ffd, pb->fs->path, pool));
10353 /* If the repository isn't a new enough format, we don't support packing.
10354 Return a friendly error to that effect. */
10355 if (ffd.format < SVN_FS_FS__MIN_PACKED_FORMAT)
10356 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
10357 _("FSFS format (%d) too old to pack; please upgrade the filesystem."),
10360 /* If we aren't using sharding, we can't do any packing, so quit. */
10361 if (!ffd.max_files_per_dir)
10362 return SVN_NO_ERROR;
10364 SVN_ERR(read_min_unpacked_rev(&ffd.min_unpacked_rev,
10365 path_min_unpacked_rev(pb->fs, pool),
10368 SVN_ERR(get_youngest(&youngest, pb->fs->path, pool));
10369 completed_shards = (youngest + 1) / ffd.max_files_per_dir;
10371 /* See if we've already completed all possible shards thus far. */
10372 if (ffd.min_unpacked_rev == (completed_shards * ffd.max_files_per_dir))
10373 return SVN_NO_ERROR;
10375 rev_data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, pool);
10376 if (ffd.format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
10377 revprops_data_path = svn_dirent_join(pb->fs->path, PATH_REVPROPS_DIR,
10380 iterpool = svn_pool_create(pool);
10381 for (i = ffd.min_unpacked_rev / ffd.max_files_per_dir;
10382 i < completed_shards;
10385 svn_pool_clear(iterpool);
10387 if (pb->cancel_func)
10388 SVN_ERR(pb->cancel_func(pb->cancel_baton));
10390 SVN_ERR(pack_shard(rev_data_path, revprops_data_path,
10391 pb->fs->path, i, ffd.max_files_per_dir,
10392 ffd.revprop_pack_size,
10393 ffd.compress_packed_revprops
10394 ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
10395 : SVN_DELTA_COMPRESSION_LEVEL_NONE,
10396 pb->notify_func, pb->notify_baton,
10397 pb->cancel_func, pb->cancel_baton, iterpool));
10400 svn_pool_destroy(iterpool);
10401 return SVN_NO_ERROR;
10405 svn_fs_fs__pack(svn_fs_t *fs,
10406 svn_fs_pack_notify_t notify_func,
10407 void *notify_baton,
10408 svn_cancel_func_t cancel_func,
10409 void *cancel_baton,
10412 struct pack_baton pb = { 0 };
10414 pb.notify_func = notify_func;
10415 pb.notify_baton = notify_baton;
10416 pb.cancel_func = cancel_func;
10417 pb.cancel_baton = cancel_baton;
10418 return svn_fs_fs__with_write_lock(fs, pack_body, &pb, pool);
10424 /* Baton type expected by verify_walker(). The purpose is to reuse open
10425 * rev / pack file handles between calls. Its contents need to be cleaned
10426 * periodically to limit resource usage.
10428 typedef struct verify_walker_baton_t
10430 /* number of calls to verify_walker() since the last clean */
10431 int iteration_count;
10433 /* number of files opened since the last clean */
10436 /* progress notification callback to invoke periodically (may be NULL) */
10437 svn_fs_progress_notify_func_t notify_func;
10439 /* baton to use with NOTIFY_FUNC */
10440 void *notify_baton;
10442 /* remember the last revision for which we called notify_func */
10443 svn_revnum_t last_notified_revision;
10445 /* current file handle (or NULL) */
10446 apr_file_t *file_hint;
10448 /* corresponding revision (or SVN_INVALID_REVNUM) */
10449 svn_revnum_t rev_hint;
10451 /* pool to use for the file handles etc. */
10453 } verify_walker_baton_t;
10455 /* Used by svn_fs_fs__verify().
10456 Implements svn_fs_fs__walk_rep_reference().walker. */
10457 static svn_error_t *
10458 verify_walker(representation_t *rep,
10461 apr_pool_t *scratch_pool)
10463 struct rep_state *rs;
10464 struct rep_args *rep_args;
10468 verify_walker_baton_t *walker_baton = baton;
10469 apr_file_t * previous_file;
10471 /* notify and free resources periodically */
10472 if ( walker_baton->iteration_count > 1000
10473 || walker_baton->file_count > 16)
10475 if ( walker_baton->notify_func
10476 && rep->revision != walker_baton->last_notified_revision)
10478 walker_baton->notify_func(rep->revision,
10479 walker_baton->notify_baton,
10481 walker_baton->last_notified_revision = rep->revision;
10484 svn_pool_clear(walker_baton->pool);
10486 walker_baton->iteration_count = 0;
10487 walker_baton->file_count = 0;
10488 walker_baton->file_hint = NULL;
10489 walker_baton->rev_hint = SVN_INVALID_REVNUM;
10492 /* access the repo data */
10493 previous_file = walker_baton->file_hint;
10494 SVN_ERR(create_rep_state(&rs, &rep_args, &walker_baton->file_hint,
10495 &walker_baton->rev_hint, rep, fs,
10496 walker_baton->pool));
10498 /* update resource usage counters */
10499 walker_baton->iteration_count++;
10500 if (previous_file != walker_baton->file_hint)
10501 walker_baton->file_count++;
10505 /* ### Should this be using read_rep_line() directly? */
10506 SVN_ERR(create_rep_state(&rs, &rep_args, NULL, NULL, rep, fs,
10510 return SVN_NO_ERROR;
10514 svn_fs_fs__verify(svn_fs_t *fs,
10515 svn_revnum_t start,
10517 svn_fs_progress_notify_func_t notify_func,
10518 void *notify_baton,
10519 svn_cancel_func_t cancel_func,
10520 void *cancel_baton,
10523 fs_fs_data_t *ffd = fs->fsap_data;
10524 svn_boolean_t exists;
10525 svn_revnum_t youngest = ffd->youngest_rev_cache; /* cache is current */
10527 if (ffd->format < SVN_FS_FS__MIN_REP_SHARING_FORMAT)
10528 return SVN_NO_ERROR;
10530 /* Input validation. */
10531 if (! SVN_IS_VALID_REVNUM(start))
10533 if (! SVN_IS_VALID_REVNUM(end))
10535 SVN_ERR(ensure_revision_exists(fs, start, pool));
10536 SVN_ERR(ensure_revision_exists(fs, end, pool));
10538 /* rep-cache verification. */
10539 SVN_ERR(svn_fs_fs__exists_rep_cache(&exists, fs, pool));
10542 /* provide a baton to allow the reuse of open file handles between
10543 iterations (saves 2/3 of OS level file operations). */
10544 verify_walker_baton_t *baton = apr_pcalloc(pool, sizeof(*baton));
10545 baton->rev_hint = SVN_INVALID_REVNUM;
10546 baton->pool = svn_pool_create(pool);
10547 baton->last_notified_revision = SVN_INVALID_REVNUM;
10548 baton->notify_func = notify_func;
10549 baton->notify_baton = notify_baton;
10551 /* tell the user that we are now ready to do *something* */
10553 notify_func(SVN_INVALID_REVNUM, notify_baton, baton->pool);
10555 /* Do not attempt to walk the rep-cache database if its file does
10556 not exist, since doing so would create it --- which may confuse
10557 the administrator. Don't take any lock. */
10558 SVN_ERR(svn_fs_fs__walk_rep_reference(fs, start, end,
10559 verify_walker, baton,
10560 cancel_func, cancel_baton,
10563 /* walker resource cleanup */
10564 svn_pool_destroy(baton->pool);
10567 return SVN_NO_ERROR;
10573 /* Like svn_io_dir_file_copy(), but doesn't copy files that exist at
10574 * the destination and do not differ in terms of kind, size, and mtime. */
10575 static svn_error_t *
10576 hotcopy_io_dir_file_copy(const char *src_path,
10577 const char *dst_path,
10579 apr_pool_t *scratch_pool)
10581 const svn_io_dirent2_t *src_dirent;
10582 const svn_io_dirent2_t *dst_dirent;
10583 const char *src_target;
10584 const char *dst_target;
10586 /* Does the destination already exist? If not, we must copy it. */
10587 dst_target = svn_dirent_join(dst_path, file, scratch_pool);
10588 SVN_ERR(svn_io_stat_dirent2(&dst_dirent, dst_target, FALSE, TRUE,
10589 scratch_pool, scratch_pool));
10590 if (dst_dirent->kind != svn_node_none)
10592 /* If the destination's stat information indicates that the file
10593 * is equal to the source, don't bother copying the file again. */
10594 src_target = svn_dirent_join(src_path, file, scratch_pool);
10595 SVN_ERR(svn_io_stat_dirent2(&src_dirent, src_target, FALSE, FALSE,
10596 scratch_pool, scratch_pool));
10597 if (src_dirent->kind == dst_dirent->kind &&
10598 src_dirent->special == dst_dirent->special &&
10599 src_dirent->filesize == dst_dirent->filesize &&
10600 src_dirent->mtime <= dst_dirent->mtime)
10601 return SVN_NO_ERROR;
10604 return svn_error_trace(svn_io_dir_file_copy(src_path, dst_path, file,
10608 /* Set *NAME_P to the UTF-8 representation of directory entry NAME.
10609 * NAME is in the internal encoding used by APR; PARENT is in
10610 * UTF-8 and in internal (not local) style.
10612 * Use PARENT only for generating an error string if the conversion
10613 * fails because NAME could not be represented in UTF-8. In that
10614 * case, return a two-level error in which the outer error's message
10615 * mentions PARENT, but the inner error's message does not mention
10616 * NAME (except possibly in hex) since NAME may not be printable.
10617 * Such a compound error at least allows the user to go looking in the
10618 * right directory for the problem.
10620 * If there is any other error, just return that error directly.
10622 * If there is any error, the effect on *NAME_P is undefined.
10624 * *NAME_P and NAME may refer to the same storage.
10626 static svn_error_t *
10627 entry_name_to_utf8(const char **name_p,
10629 const char *parent,
10632 svn_error_t *err = svn_path_cstring_to_utf8(name_p, name, pool);
10633 if (err && err->apr_err == APR_EINVAL)
10635 return svn_error_createf(err->apr_err, err,
10636 _("Error converting entry "
10637 "in directory '%s' to UTF-8"),
10638 svn_dirent_local_style(parent, pool));
10643 /* Like svn_io_copy_dir_recursively() but doesn't copy regular files that
10644 * exist in the destination and do not differ from the source in terms of
10645 * kind, size, and mtime. */
10646 static svn_error_t *
10647 hotcopy_io_copy_dir_recursively(const char *src,
10648 const char *dst_parent,
10649 const char *dst_basename,
10650 svn_boolean_t copy_perms,
10651 svn_cancel_func_t cancel_func,
10652 void *cancel_baton,
10655 svn_node_kind_t kind;
10656 apr_status_t status;
10657 const char *dst_path;
10658 apr_dir_t *this_dir;
10659 apr_finfo_t this_entry;
10660 apr_int32_t flags = APR_FINFO_TYPE | APR_FINFO_NAME;
10662 /* Make a subpool for recursion */
10663 apr_pool_t *subpool = svn_pool_create(pool);
10665 /* The 'dst_path' is simply dst_parent/dst_basename */
10666 dst_path = svn_dirent_join(dst_parent, dst_basename, pool);
10668 /* Sanity checks: SRC and DST_PARENT are directories, and
10669 DST_BASENAME doesn't already exist in DST_PARENT. */
10670 SVN_ERR(svn_io_check_path(src, &kind, subpool));
10671 if (kind != svn_node_dir)
10672 return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL,
10673 _("Source '%s' is not a directory"),
10674 svn_dirent_local_style(src, pool));
10676 SVN_ERR(svn_io_check_path(dst_parent, &kind, subpool));
10677 if (kind != svn_node_dir)
10678 return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL,
10679 _("Destination '%s' is not a directory"),
10680 svn_dirent_local_style(dst_parent, pool));
10682 SVN_ERR(svn_io_check_path(dst_path, &kind, subpool));
10684 /* Create the new directory. */
10685 /* ### TODO: copy permissions (needs apr_file_attrs_get()) */
10686 SVN_ERR(svn_io_make_dir_recursively(dst_path, pool));
10688 /* Loop over the dirents in SRC. ('.' and '..' are auto-excluded) */
10689 SVN_ERR(svn_io_dir_open(&this_dir, src, subpool));
10691 for (status = apr_dir_read(&this_entry, flags, this_dir);
10692 status == APR_SUCCESS;
10693 status = apr_dir_read(&this_entry, flags, this_dir))
10695 if ((this_entry.name[0] == '.')
10696 && ((this_entry.name[1] == '\0')
10697 || ((this_entry.name[1] == '.')
10698 && (this_entry.name[2] == '\0'))))
10704 const char *entryname_utf8;
10707 SVN_ERR(cancel_func(cancel_baton));
10709 SVN_ERR(entry_name_to_utf8(&entryname_utf8, this_entry.name,
10711 if (this_entry.filetype == APR_REG) /* regular file */
10713 SVN_ERR(hotcopy_io_dir_file_copy(src, dst_path, entryname_utf8,
10716 else if (this_entry.filetype == APR_LNK) /* symlink */
10718 const char *src_target = svn_dirent_join(src, entryname_utf8,
10720 const char *dst_target = svn_dirent_join(dst_path,
10723 SVN_ERR(svn_io_copy_link(src_target, dst_target,
10726 else if (this_entry.filetype == APR_DIR) /* recurse */
10728 const char *src_target;
10730 /* Prevent infinite recursion by filtering off our
10731 newly created destination path. */
10732 if (strcmp(src, dst_parent) == 0
10733 && strcmp(entryname_utf8, dst_basename) == 0)
10736 src_target = svn_dirent_join(src, entryname_utf8, subpool);
10737 SVN_ERR(hotcopy_io_copy_dir_recursively(src_target,
10745 /* ### support other APR node types someday?? */
10750 if (! (APR_STATUS_IS_ENOENT(status)))
10751 return svn_error_wrap_apr(status, _("Can't read directory '%s'"),
10752 svn_dirent_local_style(src, pool));
10754 status = apr_dir_close(this_dir);
10756 return svn_error_wrap_apr(status, _("Error closing directory '%s'"),
10757 svn_dirent_local_style(src, pool));
10759 /* Free any memory used by recursion */
10760 svn_pool_destroy(subpool);
10762 return SVN_NO_ERROR;
10765 /* Copy an un-packed revision or revprop file for revision REV from SRC_SUBDIR
10766 * to DST_SUBDIR. Assume a sharding layout based on MAX_FILES_PER_DIR.
10767 * Use SCRATCH_POOL for temporary allocations. */
10768 static svn_error_t *
10769 hotcopy_copy_shard_file(const char *src_subdir,
10770 const char *dst_subdir,
10772 int max_files_per_dir,
10773 apr_pool_t *scratch_pool)
10775 const char *src_subdir_shard = src_subdir,
10776 *dst_subdir_shard = dst_subdir;
10778 if (max_files_per_dir)
10780 const char *shard = apr_psprintf(scratch_pool, "%ld",
10781 rev / max_files_per_dir);
10782 src_subdir_shard = svn_dirent_join(src_subdir, shard, scratch_pool);
10783 dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool);
10785 if (rev % max_files_per_dir == 0)
10787 SVN_ERR(svn_io_make_dir_recursively(dst_subdir_shard, scratch_pool));
10788 SVN_ERR(svn_io_copy_perms(dst_subdir, dst_subdir_shard,
10793 SVN_ERR(hotcopy_io_dir_file_copy(src_subdir_shard, dst_subdir_shard,
10794 apr_psprintf(scratch_pool, "%ld", rev),
10796 return SVN_NO_ERROR;
10800 /* Copy a packed shard containing revision REV, and which contains
10801 * MAX_FILES_PER_DIR revisions, from SRC_FS to DST_FS.
10802 * Update *DST_MIN_UNPACKED_REV in case the shard is new in DST_FS.
10803 * Do not re-copy data which already exists in DST_FS.
10804 * Use SCRATCH_POOL for temporary allocations. */
10805 static svn_error_t *
10806 hotcopy_copy_packed_shard(svn_revnum_t *dst_min_unpacked_rev,
10810 int max_files_per_dir,
10811 apr_pool_t *scratch_pool)
10813 const char *src_subdir;
10814 const char *dst_subdir;
10815 const char *packed_shard;
10816 const char *src_subdir_packed_shard;
10817 svn_revnum_t revprop_rev;
10818 apr_pool_t *iterpool;
10819 fs_fs_data_t *src_ffd = src_fs->fsap_data;
10821 /* Copy the packed shard. */
10822 src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, scratch_pool);
10823 dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, scratch_pool);
10824 packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD,
10825 rev / max_files_per_dir);
10826 src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard,
10828 SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard,
10829 dst_subdir, packed_shard,
10830 TRUE /* copy_perms */,
10831 NULL /* cancel_func */, NULL,
10834 /* Copy revprops belonging to revisions in this pack. */
10835 src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, scratch_pool);
10836 dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, scratch_pool);
10838 if ( src_ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT
10839 || src_ffd->min_unpacked_rev < rev + max_files_per_dir)
10841 /* copy unpacked revprops rev by rev */
10842 iterpool = svn_pool_create(scratch_pool);
10843 for (revprop_rev = rev;
10844 revprop_rev < rev + max_files_per_dir;
10847 svn_pool_clear(iterpool);
10849 SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir,
10850 revprop_rev, max_files_per_dir,
10853 svn_pool_destroy(iterpool);
10857 /* revprop for revision 0 will never be packed */
10859 SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir,
10860 0, max_files_per_dir,
10863 /* packed revprops folder */
10864 packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD,
10865 rev / max_files_per_dir);
10866 src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard,
10868 SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard,
10869 dst_subdir, packed_shard,
10870 TRUE /* copy_perms */,
10871 NULL /* cancel_func */, NULL,
10875 /* If necessary, update the min-unpacked rev file in the hotcopy. */
10876 if (*dst_min_unpacked_rev < rev + max_files_per_dir)
10878 *dst_min_unpacked_rev = rev + max_files_per_dir;
10879 SVN_ERR(write_revnum_file(dst_fs->path, PATH_MIN_UNPACKED_REV,
10880 *dst_min_unpacked_rev,
10884 return SVN_NO_ERROR;
10887 /* If NEW_YOUNGEST is younger than *DST_YOUNGEST, update the 'current'
10888 * file in DST_FS and set *DST_YOUNGEST to NEW_YOUNGEST.
10889 * Use SCRATCH_POOL for temporary allocations. */
10890 static svn_error_t *
10891 hotcopy_update_current(svn_revnum_t *dst_youngest,
10893 svn_revnum_t new_youngest,
10894 apr_pool_t *scratch_pool)
10896 char next_node_id[MAX_KEY_SIZE] = "0";
10897 char next_copy_id[MAX_KEY_SIZE] = "0";
10898 fs_fs_data_t *dst_ffd = dst_fs->fsap_data;
10900 if (*dst_youngest >= new_youngest)
10901 return SVN_NO_ERROR;
10903 /* If necessary, get new current next_node and next_copy IDs. */
10904 if (dst_ffd->format < SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT)
10906 apr_off_t root_offset;
10907 apr_file_t *rev_file;
10909 if (dst_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
10910 SVN_ERR(update_min_unpacked_rev(dst_fs, scratch_pool));
10912 SVN_ERR(open_pack_or_rev_file(&rev_file, dst_fs, new_youngest,
10914 SVN_ERR(get_root_changes_offset(&root_offset, NULL, rev_file,
10915 dst_fs, new_youngest, scratch_pool));
10916 SVN_ERR(recover_find_max_ids(dst_fs, new_youngest, rev_file,
10917 root_offset, next_node_id, next_copy_id,
10919 SVN_ERR(svn_io_file_close(rev_file, scratch_pool));
10922 /* Update 'current'. */
10923 SVN_ERR(write_current(dst_fs, new_youngest, next_node_id, next_copy_id,
10926 *dst_youngest = new_youngest;
10928 return SVN_NO_ERROR;
10932 /* Remove revision or revprop files between START_REV (inclusive) and
10933 * END_REV (non-inclusive) from folder DST_SUBDIR in DST_FS. Assume
10934 * sharding as per MAX_FILES_PER_DIR.
10935 * Use SCRATCH_POOL for temporary allocations. */
10936 static svn_error_t *
10937 hotcopy_remove_files(svn_fs_t *dst_fs,
10938 const char *dst_subdir,
10939 svn_revnum_t start_rev,
10940 svn_revnum_t end_rev,
10941 int max_files_per_dir,
10942 apr_pool_t *scratch_pool)
10945 const char *dst_subdir_shard;
10947 apr_pool_t *iterpool;
10949 /* Pre-compute paths for initial shard. */
10950 shard = apr_psprintf(scratch_pool, "%ld", start_rev / max_files_per_dir);
10951 dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool);
10953 iterpool = svn_pool_create(scratch_pool);
10954 for (rev = start_rev; rev < end_rev; rev++)
10957 svn_pool_clear(iterpool);
10959 /* If necessary, update paths for shard. */
10960 if (rev != start_rev && rev % max_files_per_dir == 0)
10962 shard = apr_psprintf(iterpool, "%ld", rev / max_files_per_dir);
10963 dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool);
10966 /* remove files for REV */
10967 path = svn_dirent_join(dst_subdir_shard,
10968 apr_psprintf(iterpool, "%ld", rev),
10971 /* Make the rev file writable and remove it. */
10972 SVN_ERR(svn_io_set_file_read_write(path, TRUE, iterpool));
10973 SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
10976 svn_pool_destroy(iterpool);
10978 return SVN_NO_ERROR;
10981 /* Remove revisions between START_REV (inclusive) and END_REV (non-inclusive)
10982 * from DST_FS. Assume sharding as per MAX_FILES_PER_DIR.
10983 * Use SCRATCH_POOL for temporary allocations. */
10984 static svn_error_t *
10985 hotcopy_remove_rev_files(svn_fs_t *dst_fs,
10986 svn_revnum_t start_rev,
10987 svn_revnum_t end_rev,
10988 int max_files_per_dir,
10989 apr_pool_t *scratch_pool)
10991 SVN_ERR_ASSERT(start_rev <= end_rev);
10992 SVN_ERR(hotcopy_remove_files(dst_fs,
10993 svn_dirent_join(dst_fs->path,
10996 start_rev, end_rev,
10997 max_files_per_dir, scratch_pool));
10999 return SVN_NO_ERROR;
11002 /* Remove revision properties between START_REV (inclusive) and END_REV
11003 * (non-inclusive) from DST_FS. Assume sharding as per MAX_FILES_PER_DIR.
11004 * Use SCRATCH_POOL for temporary allocations. Revision 0 revprops will
11005 * not be deleted. */
11006 static svn_error_t *
11007 hotcopy_remove_revprop_files(svn_fs_t *dst_fs,
11008 svn_revnum_t start_rev,
11009 svn_revnum_t end_rev,
11010 int max_files_per_dir,
11011 apr_pool_t *scratch_pool)
11013 SVN_ERR_ASSERT(start_rev <= end_rev);
11015 /* don't delete rev 0 props */
11016 SVN_ERR(hotcopy_remove_files(dst_fs,
11017 svn_dirent_join(dst_fs->path,
11020 start_rev ? start_rev : 1, end_rev,
11021 max_files_per_dir, scratch_pool));
11023 return SVN_NO_ERROR;
11026 /* Verify that DST_FS is a suitable destination for an incremental
11027 * hotcopy from SRC_FS. */
11028 static svn_error_t *
11029 hotcopy_incremental_check_preconditions(svn_fs_t *src_fs,
11033 fs_fs_data_t *src_ffd = src_fs->fsap_data;
11034 fs_fs_data_t *dst_ffd = dst_fs->fsap_data;
11036 /* We only support incremental hotcopy between the same format. */
11037 if (src_ffd->format != dst_ffd->format)
11038 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
11039 _("The FSFS format (%d) of the hotcopy source does not match the "
11040 "FSFS format (%d) of the hotcopy destination; please upgrade "
11041 "both repositories to the same format"),
11042 src_ffd->format, dst_ffd->format);
11044 /* Make sure the UUID of source and destination match up.
11045 * We don't want to copy over a different repository. */
11046 if (strcmp(src_fs->uuid, dst_fs->uuid) != 0)
11047 return svn_error_create(SVN_ERR_RA_UUID_MISMATCH, NULL,
11048 _("The UUID of the hotcopy source does "
11049 "not match the UUID of the hotcopy "
11052 /* Also require same shard size. */
11053 if (src_ffd->max_files_per_dir != dst_ffd->max_files_per_dir)
11054 return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
11055 _("The sharding layout configuration "
11056 "of the hotcopy source does not match "
11057 "the sharding layout configuration of "
11058 "the hotcopy destination"));
11059 return SVN_NO_ERROR;
11062 /* Remove folder PATH. Ignore errors due to the sub-tree not being empty.
11063 * CANCEL_FUNC and CANCEL_BATON do the usual thing.
11064 * Use POOL for temporary allocations.
11066 static svn_error_t *
11067 remove_folder(const char *path,
11068 svn_cancel_func_t cancel_func,
11069 void *cancel_baton,
11072 svn_error_t *err = svn_io_remove_dir2(path, TRUE,
11073 cancel_func, cancel_baton, pool);
11075 if (err && APR_STATUS_IS_ENOTEMPTY(err->apr_err))
11077 svn_error_clear(err);
11078 err = SVN_NO_ERROR;
11081 return svn_error_trace(err);
11084 /* Baton for hotcopy_body(). */
11085 struct hotcopy_body_baton {
11088 svn_boolean_t incremental;
11089 svn_cancel_func_t cancel_func;
11090 void *cancel_baton;
11091 } hotcopy_body_baton;
11093 /* Perform a hotcopy, either normal or incremental.
11095 * Normal hotcopy assumes that the destination exists as an empty
11096 * directory. It behaves like an incremental hotcopy except that
11097 * none of the copied files already exist in the destination.
11099 * An incremental hotcopy copies only changed or new files to the destination,
11100 * and removes files from the destination no longer present in the source.
11101 * While the incremental hotcopy is running, readers should still be able
11102 * to access the destintation repository without error and should not see
11103 * revisions currently in progress of being copied. Readers are able to see
11104 * new fully copied revisions even if the entire incremental hotcopy procedure
11105 * has not yet completed.
11107 * Writers are blocked out completely during the entire incremental hotcopy
11108 * process to ensure consistency. This function assumes that the repository
11109 * write-lock is held.
11111 static svn_error_t *
11112 hotcopy_body(void *baton, apr_pool_t *pool)
11114 struct hotcopy_body_baton *hbb = baton;
11115 svn_fs_t *src_fs = hbb->src_fs;
11116 fs_fs_data_t *src_ffd = src_fs->fsap_data;
11117 svn_fs_t *dst_fs = hbb->dst_fs;
11118 fs_fs_data_t *dst_ffd = dst_fs->fsap_data;
11119 int max_files_per_dir = src_ffd->max_files_per_dir;
11120 svn_boolean_t incremental = hbb->incremental;
11121 svn_cancel_func_t cancel_func = hbb->cancel_func;
11122 void* cancel_baton = hbb->cancel_baton;
11123 svn_revnum_t src_youngest;
11124 svn_revnum_t dst_youngest;
11126 svn_revnum_t src_min_unpacked_rev;
11127 svn_revnum_t dst_min_unpacked_rev;
11128 const char *src_subdir;
11129 const char *dst_subdir;
11130 const char *revprop_src_subdir;
11131 const char *revprop_dst_subdir;
11132 apr_pool_t *iterpool;
11133 svn_node_kind_t kind;
11135 /* Try to copy the config.
11137 * ### We try copying the config file before doing anything else,
11138 * ### because higher layers will abort the hotcopy if we throw
11139 * ### an error from this function, and that renders the hotcopy
11140 * ### unusable anyway. */
11141 if (src_ffd->format >= SVN_FS_FS__MIN_CONFIG_FILE)
11145 err = svn_io_dir_file_copy(src_fs->path, dst_fs->path, PATH_CONFIG,
11149 if (APR_STATUS_IS_ENOENT(err->apr_err))
11151 /* 1.6.0 to 1.6.11 did not copy the configuration file during
11152 * hotcopy. So if we're hotcopying a repository which has been
11153 * created as a hotcopy itself, it's possible that fsfs.conf
11154 * does not exist. Ask the user to re-create it.
11156 * ### It would be nice to make this a non-fatal error,
11157 * ### but this function does not get an svn_fs_t object
11158 * ### so we have no way of just printing a warning via
11159 * ### the fs->warning() callback. */
11162 const char *src_abspath;
11163 const char *dst_abspath;
11164 const char *config_relpath;
11167 config_relpath = svn_dirent_join(src_fs->path, PATH_CONFIG, pool);
11168 err2 = svn_dirent_get_absolute(&src_abspath, src_fs->path, pool);
11170 return svn_error_trace(svn_error_compose_create(err, err2));
11171 err2 = svn_dirent_get_absolute(&dst_abspath, dst_fs->path, pool);
11173 return svn_error_trace(svn_error_compose_create(err, err2));
11175 /* ### hack: strip off the 'db/' directory from paths so
11176 * ### they make sense to the user */
11177 src_abspath = svn_dirent_dirname(src_abspath, pool);
11178 dst_abspath = svn_dirent_dirname(dst_abspath, pool);
11180 msg = apr_psprintf(pool,
11181 _("Failed to create hotcopy at '%s'. "
11182 "The file '%s' is missing from the source "
11183 "repository. Please create this file, for "
11184 "instance by running 'svnadmin upgrade %s'"),
11185 dst_abspath, config_relpath, src_abspath);
11186 return svn_error_quick_wrap(err, msg);
11189 return svn_error_trace(err);
11194 SVN_ERR(cancel_func(cancel_baton));
11196 /* Find the youngest revision in the source and destination.
11197 * We only support hotcopies from sources with an equal or greater amount
11198 * of revisions than the destination.
11199 * This also catches the case where users accidentally swap the
11200 * source and destination arguments. */
11201 SVN_ERR(get_youngest(&src_youngest, src_fs->path, pool));
11204 SVN_ERR(get_youngest(&dst_youngest, dst_fs->path, pool));
11205 if (src_youngest < dst_youngest)
11206 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
11207 _("The hotcopy destination already contains more revisions "
11208 "(%lu) than the hotcopy source contains (%lu); are source "
11209 "and destination swapped?"),
11210 dst_youngest, src_youngest);
11216 SVN_ERR(cancel_func(cancel_baton));
11218 /* Copy the min unpacked rev, and read its value. */
11219 if (src_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
11221 const char *min_unpacked_rev_path;
11223 min_unpacked_rev_path = svn_dirent_join(src_fs->path,
11224 PATH_MIN_UNPACKED_REV,
11226 SVN_ERR(read_min_unpacked_rev(&src_min_unpacked_rev,
11227 min_unpacked_rev_path,
11230 min_unpacked_rev_path = svn_dirent_join(dst_fs->path,
11231 PATH_MIN_UNPACKED_REV,
11233 SVN_ERR(read_min_unpacked_rev(&dst_min_unpacked_rev,
11234 min_unpacked_rev_path,
11237 /* We only support packs coming from the hotcopy source.
11238 * The destination should not be packed independently from
11239 * the source. This also catches the case where users accidentally
11240 * swap the source and destination arguments. */
11241 if (src_min_unpacked_rev < dst_min_unpacked_rev)
11242 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
11243 _("The hotcopy destination already contains "
11244 "more packed revisions (%lu) than the "
11245 "hotcopy source contains (%lu)"),
11246 dst_min_unpacked_rev - 1,
11247 src_min_unpacked_rev - 1);
11249 SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path,
11250 PATH_MIN_UNPACKED_REV, pool));
11254 src_min_unpacked_rev = 0;
11255 dst_min_unpacked_rev = 0;
11259 SVN_ERR(cancel_func(cancel_baton));
11262 * Copy the necessary rev files.
11265 src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, pool);
11266 dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, pool);
11267 SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool));
11269 iterpool = svn_pool_create(pool);
11270 /* First, copy packed shards. */
11271 for (rev = 0; rev < src_min_unpacked_rev; rev += max_files_per_dir)
11273 svn_pool_clear(iterpool);
11276 SVN_ERR(cancel_func(cancel_baton));
11278 /* Copy the packed shard. */
11279 SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev,
11281 rev, max_files_per_dir,
11284 /* If necessary, update 'current' to the most recent packed rev,
11285 * so readers can see new revisions which arrived in this pack. */
11286 SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs,
11287 rev + max_files_per_dir - 1,
11290 /* Remove revision files which are now packed. */
11293 SVN_ERR(hotcopy_remove_rev_files(dst_fs, rev,
11294 rev + max_files_per_dir,
11295 max_files_per_dir, iterpool));
11296 SVN_ERR(hotcopy_remove_revprop_files(dst_fs, rev,
11297 rev + max_files_per_dir,
11298 max_files_per_dir, iterpool));
11301 /* Now that all revisions have moved into the pack, the original
11302 * rev dir can be removed. */
11303 SVN_ERR(remove_folder(path_rev_shard(dst_fs, rev, iterpool),
11304 cancel_func, cancel_baton, iterpool));
11306 SVN_ERR(remove_folder(path_revprops_shard(dst_fs, rev, iterpool),
11307 cancel_func, cancel_baton, iterpool));
11311 SVN_ERR(cancel_func(cancel_baton));
11313 /* Now, copy pairs of non-packed revisions and revprop files.
11314 * If necessary, update 'current' after copying all files from a shard. */
11315 SVN_ERR_ASSERT(rev == src_min_unpacked_rev);
11316 SVN_ERR_ASSERT(src_min_unpacked_rev == dst_min_unpacked_rev);
11317 revprop_src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, pool);
11318 revprop_dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, pool);
11319 SVN_ERR(svn_io_make_dir_recursively(revprop_dst_subdir, pool));
11320 for (; rev <= src_youngest; rev++)
11324 svn_pool_clear(iterpool);
11327 SVN_ERR(cancel_func(cancel_baton));
11329 /* Copy the rev file. */
11330 err = hotcopy_copy_shard_file(src_subdir, dst_subdir,
11331 rev, max_files_per_dir,
11335 if (APR_STATUS_IS_ENOENT(err->apr_err) &&
11336 src_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
11338 svn_error_clear(err);
11340 /* The source rev file does not exist. This can happen if the
11341 * source repository is being packed concurrently with this
11342 * hotcopy operation.
11344 * If the new revision is now packed, and the youngest revision
11345 * we're interested in is not inside this pack, try to copy the
11348 * If the youngest revision ended up being packed, don't try
11349 * to be smart and work around this. Just abort the hotcopy. */
11350 SVN_ERR(update_min_unpacked_rev(src_fs, pool));
11351 if (is_packed_rev(src_fs, rev))
11353 if (is_packed_rev(src_fs, src_youngest))
11354 return svn_error_createf(
11355 SVN_ERR_FS_NO_SUCH_REVISION, NULL,
11356 _("The assumed HEAD revision (%lu) of the "
11357 "hotcopy source has been packed while the "
11358 "hotcopy was in progress; please restart "
11359 "the hotcopy operation"),
11362 SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev,
11364 rev, max_files_per_dir,
11366 rev = dst_min_unpacked_rev;
11370 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
11371 _("Revision %lu disappeared from the "
11372 "hotcopy source while hotcopy was "
11373 "in progress"), rev);
11376 return svn_error_trace(err);
11379 /* Copy the revprop file. */
11380 SVN_ERR(hotcopy_copy_shard_file(revprop_src_subdir,
11381 revprop_dst_subdir,
11382 rev, max_files_per_dir,
11385 /* After completing a full shard, update 'current'. */
11386 if (max_files_per_dir && rev % max_files_per_dir == 0)
11387 SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, rev, iterpool));
11389 svn_pool_destroy(iterpool);
11392 SVN_ERR(cancel_func(cancel_baton));
11394 /* We assume that all revisions were copied now, i.e. we didn't exit the
11395 * above loop early. 'rev' was last incremented during exit of the loop. */
11396 SVN_ERR_ASSERT(rev == src_youngest + 1);
11398 /* All revisions were copied. Update 'current'. */
11399 SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, src_youngest, pool));
11401 /* Replace the locks tree.
11402 * This is racy in case readers are currently trying to list locks in
11403 * the destination. However, we need to get rid of stale locks.
11404 * This is the simplest way of doing this, so we accept this small race. */
11405 dst_subdir = svn_dirent_join(dst_fs->path, PATH_LOCKS_DIR, pool);
11406 SVN_ERR(svn_io_remove_dir2(dst_subdir, TRUE, cancel_func, cancel_baton,
11408 src_subdir = svn_dirent_join(src_fs->path, PATH_LOCKS_DIR, pool);
11409 SVN_ERR(svn_io_check_path(src_subdir, &kind, pool));
11410 if (kind == svn_node_dir)
11411 SVN_ERR(svn_io_copy_dir_recursively(src_subdir, dst_fs->path,
11412 PATH_LOCKS_DIR, TRUE,
11413 cancel_func, cancel_baton, pool));
11415 /* Now copy the node-origins cache tree. */
11416 src_subdir = svn_dirent_join(src_fs->path, PATH_NODE_ORIGINS_DIR, pool);
11417 SVN_ERR(svn_io_check_path(src_subdir, &kind, pool));
11418 if (kind == svn_node_dir)
11419 SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir, dst_fs->path,
11420 PATH_NODE_ORIGINS_DIR, TRUE,
11421 cancel_func, cancel_baton, pool));
11424 * NB: Data copied below is only read by writers, not readers.
11425 * Writers are still locked out at this point.
11428 if (dst_ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT)
11430 /* Copy the rep cache and then remove entries for revisions
11431 * younger than the destination's youngest revision. */
11432 src_subdir = svn_dirent_join(src_fs->path, REP_CACHE_DB_NAME, pool);
11433 dst_subdir = svn_dirent_join(dst_fs->path, REP_CACHE_DB_NAME, pool);
11434 SVN_ERR(svn_io_check_path(src_subdir, &kind, pool));
11435 if (kind == svn_node_file)
11437 SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, pool));
11438 SVN_ERR(svn_fs_fs__del_rep_reference(dst_fs, dst_youngest, pool));
11442 /* Copy the txn-current file. */
11443 if (dst_ffd->format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT)
11444 SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path,
11445 PATH_TXN_CURRENT, pool));
11447 /* If a revprop generation file exists in the source filesystem,
11448 * reset it to zero (since this is on a different path, it will not
11449 * overlap with data already in cache). Also, clean up stale files
11450 * used for the named atomics implementation. */
11451 SVN_ERR(svn_io_check_path(path_revprop_generation(src_fs, pool),
11453 if (kind == svn_node_file)
11454 SVN_ERR(write_revprop_generation_file(dst_fs, 0, pool));
11456 SVN_ERR(cleanup_revprop_namespace(dst_fs));
11458 /* Hotcopied FS is complete. Stamp it with a format file. */
11459 SVN_ERR(write_format(svn_dirent_join(dst_fs->path, PATH_FORMAT, pool),
11460 dst_ffd->format, max_files_per_dir, TRUE, pool));
11462 return SVN_NO_ERROR;
11466 /* Set up shared data between SRC_FS and DST_FS. */
11468 hotcopy_setup_shared_fs_data(svn_fs_t *src_fs, svn_fs_t *dst_fs)
11470 fs_fs_data_t *src_ffd = src_fs->fsap_data;
11471 fs_fs_data_t *dst_ffd = dst_fs->fsap_data;
11473 /* The common pool and mutexes are shared between src and dst filesystems.
11474 * During hotcopy we only grab the mutexes for the destination, so there
11475 * is no risk of dead-lock. We don't write to the src filesystem. Shared
11476 * data for the src_fs has already been initialised in fs_hotcopy(). */
11477 dst_ffd->shared = src_ffd->shared;
11480 /* Create an empty filesystem at DST_FS at DST_PATH with the same
11481 * configuration as SRC_FS (uuid, format, and other parameters).
11482 * After creation DST_FS has no revisions, not even revision zero. */
11483 static svn_error_t *
11484 hotcopy_create_empty_dest(svn_fs_t *src_fs,
11486 const char *dst_path,
11489 fs_fs_data_t *src_ffd = src_fs->fsap_data;
11490 fs_fs_data_t *dst_ffd = dst_fs->fsap_data;
11492 dst_fs->path = apr_pstrdup(pool, dst_path);
11494 dst_ffd->max_files_per_dir = src_ffd->max_files_per_dir;
11495 dst_ffd->config = src_ffd->config;
11496 dst_ffd->format = src_ffd->format;
11498 /* Create the revision data directories. */
11499 if (dst_ffd->max_files_per_dir)
11500 SVN_ERR(svn_io_make_dir_recursively(path_rev_shard(dst_fs, 0, pool),
11503 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path,
11504 PATH_REVS_DIR, pool),
11507 /* Create the revprops directory. */
11508 if (src_ffd->max_files_per_dir)
11509 SVN_ERR(svn_io_make_dir_recursively(path_revprops_shard(dst_fs, 0, pool),
11512 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path,
11517 /* Create the transaction directory. */
11518 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, PATH_TXNS_DIR,
11522 /* Create the protorevs directory. */
11523 if (dst_ffd->format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT)
11524 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path,
11525 PATH_TXN_PROTOS_DIR,
11529 /* Create the 'current' file. */
11530 SVN_ERR(svn_io_file_create(svn_fs_fs__path_current(dst_fs, pool),
11531 (dst_ffd->format >=
11532 SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT
11533 ? "0\n" : "0 1 1\n"),
11536 /* Create lock file and UUID. */
11537 SVN_ERR(svn_io_file_create(path_lock(dst_fs, pool), "", pool));
11538 SVN_ERR(svn_fs_fs__set_uuid(dst_fs, src_fs->uuid, pool));
11540 /* Create the min unpacked rev file. */
11541 if (dst_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT)
11542 SVN_ERR(svn_io_file_create(path_min_unpacked_rev(dst_fs, pool),
11544 /* Create the txn-current file if the repository supports
11545 the transaction sequence file. */
11546 if (dst_ffd->format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT)
11548 SVN_ERR(svn_io_file_create(path_txn_current(dst_fs, pool),
11550 SVN_ERR(svn_io_file_create(path_txn_current_lock(dst_fs, pool),
11554 dst_ffd->youngest_rev_cache = 0;
11556 hotcopy_setup_shared_fs_data(src_fs, dst_fs);
11557 SVN_ERR(svn_fs_fs__initialize_caches(dst_fs, pool));
11559 return SVN_NO_ERROR;
11563 svn_fs_fs__hotcopy(svn_fs_t *src_fs,
11565 const char *src_path,
11566 const char *dst_path,
11567 svn_boolean_t incremental,
11568 svn_cancel_func_t cancel_func,
11569 void *cancel_baton,
11572 struct hotcopy_body_baton hbb;
11575 SVN_ERR(cancel_func(cancel_baton));
11577 SVN_ERR(svn_fs_fs__open(src_fs, src_path, pool));
11581 const char *dst_format_abspath;
11582 svn_node_kind_t dst_format_kind;
11584 /* Check destination format to be sure we know how to incrementally
11585 * hotcopy to the destination FS. */
11586 dst_format_abspath = svn_dirent_join(dst_path, PATH_FORMAT, pool);
11587 SVN_ERR(svn_io_check_path(dst_format_abspath, &dst_format_kind, pool));
11588 if (dst_format_kind == svn_node_none)
11590 /* Destination doesn't exist yet. Perform a normal hotcopy to a
11591 * empty destination using the same configuration as the source. */
11592 SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path, pool));
11596 /* Check the existing repository. */
11597 SVN_ERR(svn_fs_fs__open(dst_fs, dst_path, pool));
11598 SVN_ERR(hotcopy_incremental_check_preconditions(src_fs, dst_fs,
11600 hotcopy_setup_shared_fs_data(src_fs, dst_fs);
11601 SVN_ERR(svn_fs_fs__initialize_caches(dst_fs, pool));
11606 /* Start out with an empty destination using the same configuration
11607 * as the source. */
11608 SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path, pool));
11612 SVN_ERR(cancel_func(cancel_baton));
11614 hbb.src_fs = src_fs;
11615 hbb.dst_fs = dst_fs;
11616 hbb.incremental = incremental;
11617 hbb.cancel_func = cancel_func;
11618 hbb.cancel_baton = cancel_baton;
11619 SVN_ERR(svn_fs_fs__with_write_lock(dst_fs, hotcopy_body, &hbb, pool));
11621 return SVN_NO_ERROR;