1 /* transaction.c --- transaction-related functions of FSFS
3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
20 * ====================================================================
23 #include "transaction.h"
28 #include "svn_error_codes.h"
30 #include "svn_props.h"
31 #include "svn_sorts.h"
33 #include "svn_dirent_uri.h"
40 #include "low_level.h"
41 #include "temp_serializer.h"
42 #include "cached_data.h"
44 #include "rep-cache.h"
46 #include "private/svn_fs_util.h"
47 #include "private/svn_fspath.h"
48 #include "private/svn_sorts_private.h"
49 #include "private/svn_subr_private.h"
50 #include "private/svn_string_private.h"
51 #include "../libsvn_fs/fs-loader.h"
53 #include "svn_private_config.h"
55 /* Return the name of the sha1->rep mapping file in transaction TXN_ID
56 * within FS for the given SHA1 checksum. Use POOL for allocations.
58 static APR_INLINE const char *
59 path_txn_sha1(svn_fs_t *fs,
60 const svn_fs_fs__id_part_t *txn_id,
61 const unsigned char *sha1,
64 svn_checksum_t checksum;
65 checksum.digest = sha1;
66 checksum.kind = svn_checksum_sha1;
68 return svn_dirent_join(svn_fs_fs__path_txn_dir(fs, txn_id, pool),
69 svn_checksum_to_cstring(&checksum, pool),
73 static APR_INLINE const char *
74 path_txn_changes(svn_fs_t *fs,
75 const svn_fs_fs__id_part_t *txn_id,
78 return svn_dirent_join(svn_fs_fs__path_txn_dir(fs, txn_id, pool),
82 static APR_INLINE const char *
83 path_txn_props(svn_fs_t *fs,
84 const svn_fs_fs__id_part_t *txn_id,
87 return svn_dirent_join(svn_fs_fs__path_txn_dir(fs, txn_id, pool),
88 PATH_TXN_PROPS, pool);
91 static APR_INLINE const char *
92 path_txn_next_ids(svn_fs_t *fs,
93 const svn_fs_fs__id_part_t *txn_id,
96 return svn_dirent_join(svn_fs_fs__path_txn_dir(fs, txn_id, pool),
101 /* The vtable associated with an open transaction object. */
102 static txn_vtable_t txn_vtable = {
103 svn_fs_fs__commit_txn,
104 svn_fs_fs__abort_txn,
106 svn_fs_fs__txn_proplist,
107 svn_fs_fs__change_txn_prop,
109 svn_fs_fs__change_txn_props
112 /* FSFS-specific data being attached to svn_fs_txn_t.
114 typedef struct fs_txn_data_t
116 /* Strongly typed representation of the TXN's ID member. */
117 svn_fs_fs__id_part_t txn_id;
120 const svn_fs_fs__id_part_t *
121 svn_fs_fs__txn_get_id(svn_fs_txn_t *txn)
123 fs_txn_data_t *ftd = txn->fsap_data;
127 /* Functions for working with shared transaction data. */
129 /* Return the transaction object for transaction TXN_ID from the
130 transaction list of filesystem FS (which must already be locked via the
131 txn_list_lock mutex). If the transaction does not exist in the list,
132 then create a new transaction object and return it (if CREATE_NEW is
133 true) or return NULL (otherwise). */
134 static fs_fs_shared_txn_data_t *
135 get_shared_txn(svn_fs_t *fs,
136 const svn_fs_fs__id_part_t *txn_id,
137 svn_boolean_t create_new)
139 fs_fs_data_t *ffd = fs->fsap_data;
140 fs_fs_shared_data_t *ffsd = ffd->shared;
141 fs_fs_shared_txn_data_t *txn;
143 for (txn = ffsd->txns; txn; txn = txn->next)
144 if (svn_fs_fs__id_part_eq(&txn->txn_id, txn_id))
147 if (txn || !create_new)
150 /* Use the transaction object from the (single-object) freelist,
151 if one is available, or otherwise create a new object. */
154 txn = ffsd->free_txn;
155 ffsd->free_txn = NULL;
159 apr_pool_t *subpool = svn_pool_create(ffsd->common_pool);
160 txn = apr_palloc(subpool, sizeof(*txn));
164 txn->txn_id = *txn_id;
165 txn->being_written = FALSE;
167 /* Link this transaction into the head of the list. We will typically
168 be dealing with only one active transaction at a time, so it makes
169 sense for searches through the transaction list to look at the
170 newest transactions first. */
171 txn->next = ffsd->txns;
177 /* Free the transaction object for transaction TXN_ID, and remove it
178 from the transaction list of filesystem FS (which must already be
179 locked via the txn_list_lock mutex). Do nothing if the transaction
182 free_shared_txn(svn_fs_t *fs, const svn_fs_fs__id_part_t *txn_id)
184 fs_fs_data_t *ffd = fs->fsap_data;
185 fs_fs_shared_data_t *ffsd = ffd->shared;
186 fs_fs_shared_txn_data_t *txn, *prev = NULL;
188 for (txn = ffsd->txns; txn; prev = txn, txn = txn->next)
189 if (svn_fs_fs__id_part_eq(&txn->txn_id, txn_id))
196 prev->next = txn->next;
198 ffsd->txns = txn->next;
200 /* As we typically will be dealing with one transaction after another,
201 we will maintain a single-object free list so that we can hopefully
202 keep reusing the same transaction object. */
204 ffsd->free_txn = txn;
206 svn_pool_destroy(txn->pool);
210 /* Obtain a lock on the transaction list of filesystem FS, call BODY
211 with FS, BATON, and POOL, and then unlock the transaction list.
212 Return what BODY returned. */
214 with_txnlist_lock(svn_fs_t *fs,
215 svn_error_t *(*body)(svn_fs_t *fs,
221 fs_fs_data_t *ffd = fs->fsap_data;
222 fs_fs_shared_data_t *ffsd = ffd->shared;
224 SVN_MUTEX__WITH_LOCK(ffsd->txn_list_lock,
225 body(fs, baton, pool));
231 /* A structure used by unlock_proto_rev() and unlock_proto_rev_body(),
233 struct unlock_proto_rev_baton
235 svn_fs_fs__id_part_t txn_id;
239 /* Callback used in the implementation of unlock_proto_rev(). */
241 unlock_proto_rev_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool)
243 const struct unlock_proto_rev_baton *b = baton;
244 apr_file_t *lockfile = b->lockcookie;
245 fs_fs_shared_txn_data_t *txn = get_shared_txn(fs, &b->txn_id, FALSE);
246 apr_status_t apr_err;
249 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
250 _("Can't unlock unknown transaction '%s'"),
251 svn_fs_fs__id_txn_unparse(&b->txn_id, pool));
252 if (!txn->being_written)
253 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
254 _("Can't unlock nonlocked transaction '%s'"),
255 svn_fs_fs__id_txn_unparse(&b->txn_id, pool));
257 apr_err = apr_file_unlock(lockfile);
259 return svn_error_wrap_apr
261 _("Can't unlock prototype revision lockfile for transaction '%s'"),
262 svn_fs_fs__id_txn_unparse(&b->txn_id, pool));
263 apr_err = apr_file_close(lockfile);
265 return svn_error_wrap_apr
267 _("Can't close prototype revision lockfile for transaction '%s'"),
268 svn_fs_fs__id_txn_unparse(&b->txn_id, pool));
270 txn->being_written = FALSE;
275 /* Unlock the prototype revision file for transaction TXN_ID in filesystem
276 FS using cookie LOCKCOOKIE. The original prototype revision file must
277 have been closed _before_ calling this function.
279 Perform temporary allocations in POOL. */
281 unlock_proto_rev(svn_fs_t *fs,
282 const svn_fs_fs__id_part_t *txn_id,
286 struct unlock_proto_rev_baton b;
289 b.lockcookie = lockcookie;
290 return with_txnlist_lock(fs, unlock_proto_rev_body, &b, pool);
293 /* A structure used by get_writable_proto_rev() and
294 get_writable_proto_rev_body(), which see. */
295 struct get_writable_proto_rev_baton
298 svn_fs_fs__id_part_t txn_id;
301 /* Callback used in the implementation of get_writable_proto_rev(). */
303 get_writable_proto_rev_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool)
305 const struct get_writable_proto_rev_baton *b = baton;
306 void **lockcookie = b->lockcookie;
307 fs_fs_shared_txn_data_t *txn = get_shared_txn(fs, &b->txn_id, TRUE);
309 /* First, ensure that no thread in this process (including this one)
310 is currently writing to this transaction's proto-rev file. */
311 if (txn->being_written)
312 return svn_error_createf(SVN_ERR_FS_REP_BEING_WRITTEN, NULL,
313 _("Cannot write to the prototype revision file "
314 "of transaction '%s' because a previous "
315 "representation is currently being written by "
317 svn_fs_fs__id_txn_unparse(&b->txn_id, pool));
320 /* We know that no thread in this process is writing to the proto-rev
321 file, and by extension, that no thread in this process is holding a
322 lock on the prototype revision lock file. It is therefore safe
323 for us to attempt to lock this file, to see if any other process
324 is holding a lock. */
327 apr_file_t *lockfile;
328 apr_status_t apr_err;
329 const char *lockfile_path
330 = svn_fs_fs__path_txn_proto_rev_lock(fs, &b->txn_id, pool);
332 /* Open the proto-rev lockfile, creating it if necessary, as it may
333 not exist if the transaction dates from before the lockfiles were
336 ### We'd also like to use something like svn_io_file_lock2(), but
337 that forces us to create a subpool just to be able to unlock
338 the file, which seems a waste. */
339 SVN_ERR(svn_io_file_open(&lockfile, lockfile_path,
340 APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool));
342 apr_err = apr_file_lock(lockfile,
343 APR_FLOCK_EXCLUSIVE | APR_FLOCK_NONBLOCK);
346 svn_error_clear(svn_io_file_close(lockfile, pool));
348 if (APR_STATUS_IS_EAGAIN(apr_err))
349 return svn_error_createf(SVN_ERR_FS_REP_BEING_WRITTEN, NULL,
350 _("Cannot write to the prototype revision "
351 "file of transaction '%s' because a "
352 "previous representation is currently "
353 "being written by another process"),
354 svn_fs_fs__id_txn_unparse(&b->txn_id,
357 return svn_error_wrap_apr(apr_err,
358 _("Can't get exclusive lock on file '%s'"),
359 svn_dirent_local_style(lockfile_path, pool));
362 *lockcookie = lockfile;
365 /* We've successfully locked the transaction; mark it as such. */
366 txn->being_written = TRUE;
371 /* Make sure the length ACTUAL_LENGTH of the proto-revision file PROTO_REV
372 of transaction TXN_ID in filesystem FS matches the proto-index file.
373 Trim any crash / failure related extra data from the proto-rev file.
375 If the prototype revision file is too short, we can't do much but bail out.
377 Perform all allocations in POOL. */
379 auto_truncate_proto_rev(svn_fs_t *fs,
380 apr_file_t *proto_rev,
381 apr_off_t actual_length,
382 const svn_fs_fs__id_part_t *txn_id,
385 /* Only relevant for newer FSFS formats. */
386 if (svn_fs_fs__use_log_addressing(fs))
388 /* Determine file range covered by the proto-index so far. Note that
389 we always append to both file, i.e. the last index entry also
390 corresponds to the last addition in the rev file. */
391 const char *path = svn_fs_fs__path_p2l_proto_index(fs, txn_id, pool);
393 apr_off_t indexed_length;
395 SVN_ERR(svn_fs_fs__p2l_proto_index_open(&file, path, pool));
396 SVN_ERR(svn_fs_fs__p2l_proto_index_next_offset(&indexed_length, file,
398 SVN_ERR(svn_io_file_close(file, pool));
400 /* Handle mismatches. */
401 if (indexed_length < actual_length)
402 SVN_ERR(svn_io_file_trunc(proto_rev, indexed_length, pool));
403 else if (indexed_length > actual_length)
404 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT,
406 _("p2l proto index offset %s beyond proto"
407 "rev file size %s for TXN %s"),
408 apr_off_t_toa(pool, indexed_length),
409 apr_off_t_toa(pool, actual_length),
410 svn_fs_fs__id_txn_unparse(txn_id, pool));
416 /* Get a handle to the prototype revision file for transaction TXN_ID in
417 filesystem FS, and lock it for writing. Return FILE, a file handle
418 positioned at the end of the file, and LOCKCOOKIE, a cookie that
419 should be passed to unlock_proto_rev() to unlock the file once FILE
422 If the prototype revision file is already locked, return error
423 SVN_ERR_FS_REP_BEING_WRITTEN.
425 Perform all allocations in POOL. */
427 get_writable_proto_rev(apr_file_t **file,
430 const svn_fs_fs__id_part_t *txn_id,
433 struct get_writable_proto_rev_baton b;
435 apr_off_t end_offset = 0;
437 b.lockcookie = lockcookie;
440 SVN_ERR(with_txnlist_lock(fs, get_writable_proto_rev_body, &b, pool));
442 /* Now open the prototype revision file and seek to the end. */
443 err = svn_io_file_open(file,
444 svn_fs_fs__path_txn_proto_rev(fs, txn_id, pool),
445 APR_READ | APR_WRITE | APR_BUFFERED, APR_OS_DEFAULT,
448 /* You might expect that we could dispense with the following seek
449 and achieve the same thing by opening the file using APR_APPEND.
450 Unfortunately, APR's buffered file implementation unconditionally
451 places its initial file pointer at the start of the file (even for
452 files opened with APR_APPEND), so we need this seek to reconcile
453 the APR file pointer to the OS file pointer (since we need to be
454 able to read the current file position later). */
456 err = svn_io_file_seek(*file, APR_END, &end_offset, pool);
458 /* We don't want unused sections (such as leftovers from failed delta
459 stream) in our file. If we use log addressing, we would need an
460 index entry for the unused section and that section would need to
461 be all NUL by convention. So, detect and fix those cases by truncating
462 the protorev file. */
464 err = auto_truncate_proto_rev(fs, *file, end_offset, txn_id, pool);
468 err = svn_error_compose_create(
470 unlock_proto_rev(fs, txn_id, *lockcookie, pool));
475 return svn_error_trace(err);
478 /* Callback used in the implementation of purge_shared_txn(). */
480 purge_shared_txn_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool)
482 const svn_fs_fs__id_part_t *txn_id = baton;
484 free_shared_txn(fs, txn_id);
485 svn_fs_fs__reset_txn_caches(fs);
490 /* Purge the shared data for transaction TXN_ID in filesystem FS.
491 Perform all allocations in POOL. */
493 purge_shared_txn(svn_fs_t *fs,
494 const svn_fs_fs__id_part_t *txn_id,
497 return with_txnlist_lock(fs, purge_shared_txn_body, txn_id, pool);
502 svn_fs_fs__put_node_revision(svn_fs_t *fs,
503 const svn_fs_id_t *id,
504 node_revision_t *noderev,
505 svn_boolean_t fresh_txn_root,
508 fs_fs_data_t *ffd = fs->fsap_data;
509 apr_file_t *noderev_file;
511 noderev->is_fresh_txn_root = fresh_txn_root;
513 if (! svn_fs_fs__id_is_txn(id))
514 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
515 _("Attempted to write to non-transaction '%s'"),
516 svn_fs_fs__id_unparse(id, pool)->data);
518 SVN_ERR(svn_io_file_open(&noderev_file,
519 svn_fs_fs__path_txn_node_rev(fs, id, pool),
520 APR_WRITE | APR_CREATE | APR_TRUNCATE
521 | APR_BUFFERED, APR_OS_DEFAULT, pool));
523 SVN_ERR(svn_fs_fs__write_noderev(svn_stream_from_aprfile2(noderev_file, TRUE,
525 noderev, ffd->format,
526 svn_fs_fs__fs_supports_mergeinfo(fs),
529 SVN_ERR(svn_io_file_close(noderev_file, pool));
534 /* For the in-transaction NODEREV within FS, write the sha1->rep mapping
535 * file in the respective transaction, if rep sharing has been enabled etc.
536 * Use SCATCH_POOL for temporary allocations.
539 store_sha1_rep_mapping(svn_fs_t *fs,
540 node_revision_t *noderev,
541 apr_pool_t *scratch_pool)
543 fs_fs_data_t *ffd = fs->fsap_data;
545 /* if rep sharing has been enabled and the noderev has a data rep and
546 * its SHA-1 is known, store the rep struct under its SHA1. */
547 if ( ffd->rep_sharing_allowed
549 && noderev->data_rep->has_sha1)
551 apr_file_t *rep_file;
552 const char *file_name = path_txn_sha1(fs,
553 &noderev->data_rep->txn_id,
554 noderev->data_rep->sha1_digest,
556 svn_stringbuf_t *rep_string
557 = svn_fs_fs__unparse_representation(noderev->data_rep,
559 (noderev->kind == svn_node_dir),
560 scratch_pool, scratch_pool);
561 SVN_ERR(svn_io_file_open(&rep_file, file_name,
562 APR_WRITE | APR_CREATE | APR_TRUNCATE
563 | APR_BUFFERED, APR_OS_DEFAULT, scratch_pool));
565 SVN_ERR(svn_io_file_write_full(rep_file, rep_string->data,
566 rep_string->len, NULL, scratch_pool));
568 SVN_ERR(svn_io_file_close(rep_file, scratch_pool));
575 unparse_dir_entry(svn_fs_dirent_t *dirent,
576 svn_stream_t *stream,
580 svn_string_t *id_str = svn_fs_fs__id_unparse(dirent->id, pool);
581 apr_size_t name_len = strlen(dirent->name);
583 /* Note that sizeof == len + 1, i.e. accounts for the space between
585 apr_size_t type_len = (dirent->kind == svn_node_file)
586 ? sizeof(SVN_FS_FS__KIND_FILE)
587 : sizeof(SVN_FS_FS__KIND_DIR);
588 apr_size_t value_len = type_len + id_str->len;
590 /* A buffer with sufficient space for
591 * - both string lines
593 * - 2 lines K/V lines containing a number each
595 char *buffer = apr_palloc(pool, name_len + value_len
597 + 2 * (2 + SVN_INT64_BUFFER_SIZE));
599 /* Now construct the value. */
602 /* The "K length(name)\n" line. */
606 p += svn__i64toa(p, name_len);
609 /* The line with the key, i.e. dir entry name. */
610 memcpy(p, dirent->name, name_len);
614 /* The "V length(type+id)\n" line. */
618 p += svn__i64toa(p, value_len);
621 /* The line with the type and ID. */
623 (dirent->kind == svn_node_file) ? SVN_FS_FS__KIND_FILE
624 : SVN_FS_FS__KIND_DIR,
628 memcpy(p, id_str->data, id_str->len);
632 /* Add the entry to the output stream. */
633 to_write = p - buffer;
634 SVN_ERR(svn_stream_write(stream, buffer, &to_write));
638 /* Write the directory given as array of dirent structs in ENTRIES to STREAM.
639 Perform temporary allocations in POOL. */
641 unparse_dir_entries(apr_array_header_t *entries,
642 svn_stream_t *stream,
645 apr_pool_t *iterpool = svn_pool_create(pool);
647 for (i = 0; i < entries->nelts; ++i)
649 svn_fs_dirent_t *dirent;
651 svn_pool_clear(iterpool);
652 dirent = APR_ARRAY_IDX(entries, i, svn_fs_dirent_t *);
653 SVN_ERR(unparse_dir_entry(dirent, stream, iterpool));
656 SVN_ERR(svn_stream_printf(stream, pool, "%s\n", SVN_HASH_TERMINATOR));
658 svn_pool_destroy(iterpool);
662 /* Return a deep copy of SOURCE and allocate it in RESULT_POOL.
664 static svn_fs_path_change2_t *
665 path_change_dup(const svn_fs_path_change2_t *source,
666 apr_pool_t *result_pool)
668 svn_fs_path_change2_t *result = apr_pmemdup(result_pool, source,
670 result->node_rev_id = svn_fs_fs__id_copy(source->node_rev_id, result_pool);
671 if (source->copyfrom_path)
672 result->copyfrom_path = apr_pstrdup(result_pool, source->copyfrom_path);
677 /* Merge the internal-use-only CHANGE into a hash of public-FS
678 svn_fs_path_change2_t CHANGED_PATHS, collapsing multiple changes into a
679 single summarical (is that real word?) change per path. DELETIONS is
680 also a path->svn_fs_path_change2_t hash and contains all the deletions
681 that got turned into a replacement. */
683 fold_change(apr_hash_t *changed_paths,
684 apr_hash_t *deletions,
685 const change_t *change)
687 apr_pool_t *pool = apr_hash_pool_get(changed_paths);
688 svn_fs_path_change2_t *old_change, *new_change;
689 const svn_string_t *path = &change->path;
690 const svn_fs_path_change2_t *info = &change->info;
692 if ((old_change = apr_hash_get(changed_paths, path->data, path->len)))
694 /* This path already exists in the hash, so we have to merge
695 this change into the already existing one. */
697 /* Sanity check: only allow NULL node revision ID in the
699 if ((! info->node_rev_id)
700 && (info->change_kind != svn_fs_path_change_reset))
701 return svn_error_create
702 (SVN_ERR_FS_CORRUPT, NULL,
703 _("Missing required node revision ID"));
705 /* Sanity check: we should be talking about the same node
706 revision ID as our last change except where the last change
708 if (info->node_rev_id
709 && (! svn_fs_fs__id_eq(old_change->node_rev_id, info->node_rev_id))
710 && (old_change->change_kind != svn_fs_path_change_delete))
711 return svn_error_create
712 (SVN_ERR_FS_CORRUPT, NULL,
713 _("Invalid change ordering: new node revision ID "
716 /* Sanity check: an add, replacement, or reset must be the first
717 thing to follow a deletion. */
718 if ((old_change->change_kind == svn_fs_path_change_delete)
719 && (! ((info->change_kind == svn_fs_path_change_replace)
720 || (info->change_kind == svn_fs_path_change_reset)
721 || (info->change_kind == svn_fs_path_change_add))))
722 return svn_error_create
723 (SVN_ERR_FS_CORRUPT, NULL,
724 _("Invalid change ordering: non-add change on deleted path"));
726 /* Sanity check: an add can't follow anything except
727 a delete or reset. */
728 if ((info->change_kind == svn_fs_path_change_add)
729 && (old_change->change_kind != svn_fs_path_change_delete)
730 && (old_change->change_kind != svn_fs_path_change_reset))
731 return svn_error_create
732 (SVN_ERR_FS_CORRUPT, NULL,
733 _("Invalid change ordering: add change on preexisting path"));
735 /* Now, merge that change in. */
736 switch (info->change_kind)
738 case svn_fs_path_change_reset:
739 /* A reset here will simply remove the path change from the
741 apr_hash_set(changed_paths, path->data, path->len, NULL);
744 case svn_fs_path_change_delete:
745 if (old_change->change_kind == svn_fs_path_change_add)
747 /* If the path was introduced in this transaction via an
748 add, and we are deleting it, just remove the path
749 altogether. (The caller will delete any child paths.) */
750 apr_hash_set(changed_paths, path->data, path->len, NULL);
752 else if (old_change->change_kind == svn_fs_path_change_replace)
754 /* A deleting a 'replace' restore the original deletion. */
755 new_change = apr_hash_get(deletions, path->data, path->len);
756 SVN_ERR_ASSERT(new_change);
757 apr_hash_set(changed_paths, path->data, path->len, new_change);
761 /* A deletion overrules a previous change (modify). */
762 new_change = path_change_dup(info, pool);
763 apr_hash_set(changed_paths, path->data, path->len, new_change);
767 case svn_fs_path_change_add:
768 case svn_fs_path_change_replace:
769 /* An add at this point must be following a previous delete,
770 so treat it just like a replace. Remember the original
771 deletion such that we are able to delete this path again
772 (the replacement may have changed node kind and id). */
773 new_change = path_change_dup(info, pool);
774 new_change->change_kind = svn_fs_path_change_replace;
776 apr_hash_set(changed_paths, path->data, path->len, new_change);
778 /* Remember the original change.
779 * Make sure to allocate the hash key in a durable pool. */
780 apr_hash_set(deletions,
781 apr_pstrmemdup(apr_hash_pool_get(deletions),
782 path->data, path->len),
783 path->len, old_change);
786 case svn_fs_path_change_modify:
788 /* If the new change modifies some attribute of the node, set
789 the corresponding flag, whether it already was set or not.
790 Note: We do not reset a flag to FALSE if a change is undone. */
792 old_change->text_mod = TRUE;
794 old_change->prop_mod = TRUE;
795 if (info->mergeinfo_mod == svn_tristate_true)
796 old_change->mergeinfo_mod = svn_tristate_true;
802 /* Add this path. The API makes no guarantees that this (new) key
803 will not be retained. Thus, we copy the key into the target pool
804 to ensure a proper lifetime. */
805 apr_hash_set(changed_paths,
806 apr_pstrmemdup(pool, path->data, path->len), path->len,
807 path_change_dup(info, pool));
813 /* Baton type to be used with process_changes(). */
814 typedef struct process_changes_baton_t
816 /* Folded list of path changes. */
817 apr_hash_t *changed_paths;
819 /* Path changes that are deletions and have been turned into
820 replacements. If those replacements get deleted again, this
821 container contains the record that we have to revert to. */
822 apr_hash_t *deletions;
823 } process_changes_baton_t;
825 /* An implementation of svn_fs_fs__change_receiver_t.
826 Examine all the changed path entries in CHANGES and store them in
827 *CHANGED_PATHS. Folding is done to remove redundant or unnecessary
828 data. Do all allocations in POOL. */
830 process_changes(void *baton_p,
832 apr_pool_t *scratch_pool)
834 process_changes_baton_t *baton = baton_p;
836 SVN_ERR(fold_change(baton->changed_paths, baton->deletions, change));
838 /* Now, if our change was a deletion or replacement, we have to
839 blow away any changes thus far on paths that are (or, were)
840 children of this path.
841 ### i won't bother with another iteration pool here -- at
842 most we talking about a few extra dups of paths into what
843 is already a temporary subpool.
846 if ((change->info.change_kind == svn_fs_path_change_delete)
847 || (change->info.change_kind == svn_fs_path_change_replace))
849 apr_hash_index_t *hi;
851 /* a potential child path must contain at least 2 more chars
852 (the path separator plus at least one char for the name).
853 Also, we should not assume that all paths have been normalized
854 i.e. some might have trailing path separators.
856 apr_ssize_t path_len = change->path.len;
857 apr_ssize_t min_child_len = path_len == 0
859 : change->path.data[path_len-1] == '/'
863 /* CAUTION: This is the inner loop of an O(n^2) algorithm.
864 The number of changes to process may be >> 1000.
865 Therefore, keep the inner loop as tight as possible.
867 for (hi = apr_hash_first(scratch_pool, baton->changed_paths);
869 hi = apr_hash_next(hi))
871 /* KEY is the path. */
874 svn_fs_path_change2_t *old_change;
875 apr_hash_this(hi, &path, &klen, (void**)&old_change);
877 /* If we come across a child of our path, remove it.
878 Call svn_fspath__skip_ancestor only if there is a chance that
879 this is actually a sub-path.
881 if (klen >= min_child_len)
885 child = svn_fspath__skip_ancestor(change->path.data, path);
886 if (child && child[0] != '\0')
888 apr_hash_set(baton->changed_paths, path, klen, NULL);
898 svn_fs_fs__txn_changes_fetch(apr_hash_t **changed_paths_p,
900 const svn_fs_fs__id_part_t *txn_id,
904 apr_hash_t *changed_paths = apr_hash_make(pool);
905 apr_pool_t *scratch_pool = svn_pool_create(pool);
906 process_changes_baton_t baton;
908 baton.changed_paths = changed_paths;
909 baton.deletions = apr_hash_make(scratch_pool);
911 SVN_ERR(svn_io_file_open(&file,
912 path_txn_changes(fs, txn_id, scratch_pool),
913 APR_READ | APR_BUFFERED, APR_OS_DEFAULT,
916 SVN_ERR(svn_fs_fs__read_changes_incrementally(
917 svn_stream_from_aprfile2(file, TRUE,
919 process_changes, &baton,
921 svn_pool_destroy(scratch_pool);
923 *changed_paths_p = changed_paths;
930 svn_fs_fs__paths_changed(apr_hash_t **changed_paths_p,
935 apr_hash_t *changed_paths = svn_hash__make(pool);
936 svn_fs_fs__changes_context_t *context;
938 apr_pool_t *iterpool = svn_pool_create(pool);
940 /* Fetch all data block-by-block. */
941 SVN_ERR(svn_fs_fs__create_changes_context(&context, fs, rev, pool));
942 while (!context->eol)
944 apr_array_header_t *changes;
947 svn_pool_clear(iterpool);
949 /* Be sure to allocate the changes in the result POOL, even though
950 we don't need the array itself afterwards. Copying the entries
951 from a temp pool to the result POOL would be expensive and saves
952 use less then 10% memory. */
953 SVN_ERR(svn_fs_fs__get_changes(&changes, context, pool, iterpool));
955 for (i = 0; i < changes->nelts; ++i)
957 change_t *change = APR_ARRAY_IDX(changes, i, change_t *);
958 apr_hash_set(changed_paths, change->path.data, change->path.len,
963 svn_pool_destroy(iterpool);
965 *changed_paths_p = changed_paths;
970 /* Copy a revision node-rev SRC into the current transaction TXN_ID in
971 the filesystem FS. This is only used to create the root of a transaction.
972 Allocations are from POOL. */
974 create_new_txn_noderev_from_rev(svn_fs_t *fs,
975 const svn_fs_fs__id_part_t *txn_id,
979 node_revision_t *noderev;
980 const svn_fs_fs__id_part_t *node_id, *copy_id;
982 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, src, pool, pool));
984 if (svn_fs_fs__id_is_txn(noderev->id))
985 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
986 _("Copying from transactions not allowed"));
988 noderev->predecessor_id = noderev->id;
989 noderev->predecessor_count++;
990 noderev->copyfrom_path = NULL;
991 noderev->copyfrom_rev = SVN_INVALID_REVNUM;
993 /* For the transaction root, the copyroot never changes. */
995 node_id = svn_fs_fs__id_node_id(noderev->id);
996 copy_id = svn_fs_fs__id_copy_id(noderev->id);
997 noderev->id = svn_fs_fs__id_txn_create(node_id, copy_id, txn_id, pool);
999 return svn_fs_fs__put_node_revision(fs, noderev->id, noderev, TRUE, pool);
1002 /* A structure used by get_and_increment_txn_key_body(). */
1003 struct get_and_increment_txn_key_baton {
1005 apr_uint64_t txn_number;
1009 /* Callback used in the implementation of create_txn_dir(). This gets
1010 the current base 36 value in PATH_TXN_CURRENT and increments it.
1011 It returns the original value by the baton. */
1012 static svn_error_t *
1013 get_and_increment_txn_key_body(void *baton, apr_pool_t *pool)
1015 struct get_and_increment_txn_key_baton *cb = baton;
1016 fs_fs_data_t *ffd = cb->fs->fsap_data;
1017 const char *txn_current_filename
1018 = svn_fs_fs__path_txn_current(cb->fs, pool);
1019 char new_id_str[SVN_INT64_BUFFER_SIZE + 1]; /* add space for a newline */
1020 apr_size_t line_length;
1022 svn_stringbuf_t *buf;
1023 SVN_ERR(svn_fs_fs__read_content(&buf, txn_current_filename, cb->pool));
1025 /* assign the current txn counter value to our result */
1026 cb->txn_number = svn__base36toui64(NULL, buf->data);
1028 /* remove trailing newlines */
1029 line_length = svn__ui64tobase36(new_id_str, cb->txn_number+1);
1030 new_id_str[line_length] = '\n';
1032 /* Increment the key and add a trailing \n to the string so the
1033 txn-current file has a newline in it. */
1034 SVN_ERR(svn_io_write_atomic2(txn_current_filename, new_id_str,
1036 txn_current_filename /* copy_perms path */,
1037 ffd->flush_to_disk, pool));
1039 return SVN_NO_ERROR;
1042 /* Create a unique directory for a transaction in FS based on revision REV.
1043 Return the ID for this transaction in *ID_P and *TXN_ID. Use a sequence
1044 value in the transaction ID to prevent reuse of transaction IDs. */
1045 static svn_error_t *
1046 create_txn_dir(const char **id_p,
1047 svn_fs_fs__id_part_t *txn_id,
1052 struct get_and_increment_txn_key_baton cb;
1053 const char *txn_dir;
1055 /* Get the current transaction sequence value, which is a base-36
1056 number, from the txn-current file, and write an
1057 incremented value back out to the file. Place the revision
1058 number the transaction is based off into the transaction id. */
1061 SVN_ERR(svn_fs_fs__with_txn_current_lock(fs,
1062 get_and_increment_txn_key_body,
1065 txn_id->revision = rev;
1066 txn_id->number = cb.txn_number;
1068 *id_p = svn_fs_fs__id_txn_unparse(txn_id, pool);
1069 txn_dir = svn_fs_fs__path_txn_dir(fs, txn_id, pool);
1071 return svn_io_dir_make(txn_dir, APR_OS_DEFAULT, pool);
1074 /* Create a unique directory for a transaction in FS based on revision
1075 REV. Return the ID for this transaction in *ID_P and *TXN_ID. This
1076 implementation is used in svn 1.4 and earlier repositories and is
1077 kept in 1.5 and greater to support the --pre-1.4-compatible and
1078 --pre-1.5-compatible repository creation options. Reused
1079 transaction IDs are possible with this implementation. */
1080 static svn_error_t *
1081 create_txn_dir_pre_1_5(const char **id_p,
1082 svn_fs_fs__id_part_t *txn_id,
1088 apr_pool_t *subpool;
1089 const char *unique_path, *prefix;
1091 /* Try to create directories named "<txndir>/<rev>-<uniqueifier>.txn". */
1092 prefix = svn_dirent_join(svn_fs_fs__path_txns_dir(fs, pool),
1093 apr_psprintf(pool, "%ld", rev), pool);
1095 subpool = svn_pool_create(pool);
1096 for (i = 1; i <= 99999; i++)
1100 svn_pool_clear(subpool);
1101 unique_path = apr_psprintf(subpool, "%s-%u" PATH_EXT_TXN, prefix, i);
1102 err = svn_io_dir_make(unique_path, APR_OS_DEFAULT, subpool);
1105 /* We succeeded. Return the basename minus the ".txn" extension. */
1106 const char *name = svn_dirent_basename(unique_path, subpool);
1107 *id_p = apr_pstrndup(pool, name,
1108 strlen(name) - strlen(PATH_EXT_TXN));
1109 SVN_ERR(svn_fs_fs__id_txn_parse(txn_id, *id_p));
1110 svn_pool_destroy(subpool);
1111 return SVN_NO_ERROR;
1113 if (! APR_STATUS_IS_EEXIST(err->apr_err))
1114 return svn_error_trace(err);
1115 svn_error_clear(err);
1118 return svn_error_createf(SVN_ERR_IO_UNIQUE_NAMES_EXHAUSTED,
1120 _("Unable to create transaction directory "
1121 "in '%s' for revision %ld"),
1122 svn_dirent_local_style(fs->path, pool),
1127 svn_fs_fs__create_txn(svn_fs_txn_t **txn_p,
1132 fs_fs_data_t *ffd = fs->fsap_data;
1135 svn_fs_id_t *root_id;
1137 txn = apr_pcalloc(pool, sizeof(*txn));
1138 ftd = apr_pcalloc(pool, sizeof(*ftd));
1140 /* Get the txn_id. */
1141 if (ffd->format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT)
1142 SVN_ERR(create_txn_dir(&txn->id, &ftd->txn_id, fs, rev, pool));
1144 SVN_ERR(create_txn_dir_pre_1_5(&txn->id, &ftd->txn_id, fs, rev, pool));
1147 txn->base_rev = rev;
1149 txn->vtable = &txn_vtable;
1150 txn->fsap_data = ftd;
1153 /* Create a new root node for this transaction. */
1154 SVN_ERR(svn_fs_fs__rev_get_root(&root_id, fs, rev, pool, pool));
1155 SVN_ERR(create_new_txn_noderev_from_rev(fs, &ftd->txn_id, root_id, pool));
1157 /* Create an empty rev file. */
1158 SVN_ERR(svn_io_file_create_empty(
1159 svn_fs_fs__path_txn_proto_rev(fs, &ftd->txn_id, pool),
1162 /* Create an empty rev-lock file. */
1163 SVN_ERR(svn_io_file_create_empty(
1164 svn_fs_fs__path_txn_proto_rev_lock(fs, &ftd->txn_id, pool),
1167 /* Create an empty changes file. */
1168 SVN_ERR(svn_io_file_create_empty(path_txn_changes(fs, &ftd->txn_id, pool),
1171 /* Create the next-ids file. */
1172 return svn_io_file_create(path_txn_next_ids(fs, &ftd->txn_id, pool),
1176 /* Store the property list for transaction TXN_ID in PROPLIST.
1177 Perform temporary allocations in POOL. */
1178 static svn_error_t *
1179 get_txn_proplist(apr_hash_t *proplist,
1181 const svn_fs_fs__id_part_t *txn_id,
1184 svn_stream_t *stream;
1187 /* Check for issue #3696. (When we find and fix the cause, we can change
1188 * this to an assertion.) */
1189 if (!txn_id || !svn_fs_fs__id_txn_used(txn_id))
1190 return svn_error_create(SVN_ERR_INCORRECT_PARAMS, NULL,
1191 _("Internal error: a null transaction id was "
1192 "passed to get_txn_proplist()"));
1194 /* Open the transaction properties file. */
1195 SVN_ERR(svn_stream_open_readonly(&stream, path_txn_props(fs, txn_id, pool),
1198 /* Read in the property list. */
1199 err = svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool);
1202 err = svn_error_compose_create(err, svn_stream_close(stream));
1203 return svn_error_quick_wrapf(err,
1204 _("malformed property list in transaction '%s'"),
1205 path_txn_props(fs, txn_id, pool));
1208 return svn_stream_close(stream);
1211 /* Save the property list PROPS as the revprops for transaction TXN_ID
1212 in FS. Perform temporary allocations in POOL. */
1213 static svn_error_t *
1214 set_txn_proplist(svn_fs_t *fs,
1215 const svn_fs_fs__id_part_t *txn_id,
1219 svn_stream_t *tmp_stream;
1220 const char *tmp_path;
1221 const char *final_path = path_txn_props(fs, txn_id, pool);
1223 /* Write the new contents into a temporary file. */
1224 SVN_ERR(svn_stream_open_unique(&tmp_stream, &tmp_path,
1225 svn_dirent_dirname(final_path, pool),
1226 svn_io_file_del_none,
1229 /* Replace the old file with the new one. */
1230 SVN_ERR(svn_hash_write2(props, tmp_stream, SVN_HASH_TERMINATOR, pool));
1231 SVN_ERR(svn_stream_close(tmp_stream));
1233 SVN_ERR(svn_io_file_rename2(tmp_path, final_path, FALSE, pool));
1235 return SVN_NO_ERROR;
1240 svn_fs_fs__change_txn_prop(svn_fs_txn_t *txn,
1242 const svn_string_t *value,
1245 apr_array_header_t *props = apr_array_make(pool, 1, sizeof(svn_prop_t));
1250 APR_ARRAY_PUSH(props, svn_prop_t) = prop;
1252 return svn_fs_fs__change_txn_props(txn, props, pool);
1256 svn_fs_fs__change_txn_props(svn_fs_txn_t *txn,
1257 const apr_array_header_t *props,
1260 fs_txn_data_t *ftd = txn->fsap_data;
1261 apr_hash_t *txn_prop = apr_hash_make(pool);
1265 err = get_txn_proplist(txn_prop, txn->fs, &ftd->txn_id, pool);
1266 /* Here - and here only - we need to deal with the possibility that the
1267 transaction property file doesn't yet exist. The rest of the
1268 implementation assumes that the file exists, but we're called to set the
1269 initial transaction properties as the transaction is being created. */
1270 if (err && (APR_STATUS_IS_ENOENT(err->apr_err)))
1271 svn_error_clear(err);
1273 return svn_error_trace(err);
1275 for (i = 0; i < props->nelts; i++)
1277 svn_prop_t *prop = &APR_ARRAY_IDX(props, i, svn_prop_t);
1279 if (svn_hash_gets(txn_prop, SVN_FS__PROP_TXN_CLIENT_DATE)
1280 && !strcmp(prop->name, SVN_PROP_REVISION_DATE))
1281 svn_hash_sets(txn_prop, SVN_FS__PROP_TXN_CLIENT_DATE,
1282 svn_string_create("1", pool));
1284 svn_hash_sets(txn_prop, prop->name, prop->value);
1287 /* Create a new version of the file and write out the new props. */
1288 /* Open the transaction properties file. */
1289 SVN_ERR(set_txn_proplist(txn->fs, &ftd->txn_id, txn_prop, pool));
1291 return SVN_NO_ERROR;
1295 svn_fs_fs__get_txn(transaction_t **txn_p,
1297 const svn_fs_fs__id_part_t *txn_id,
1301 node_revision_t *noderev;
1302 svn_fs_id_t *root_id;
1304 txn = apr_pcalloc(pool, sizeof(*txn));
1305 root_id = svn_fs_fs__id_txn_create_root(txn_id, pool);
1307 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, root_id, pool, pool));
1309 txn->root_id = svn_fs_fs__id_copy(noderev->id, pool);
1310 txn->base_id = svn_fs_fs__id_copy(noderev->predecessor_id, pool);
1315 return SVN_NO_ERROR;
1318 /* Write out the currently available next node_id NODE_ID and copy_id
1319 COPY_ID for transaction TXN_ID in filesystem FS. The next node-id is
1320 used both for creating new unique nodes for the given transaction, as
1321 well as uniquifying representations. Perform temporary allocations in
1323 static svn_error_t *
1324 write_next_ids(svn_fs_t *fs,
1325 const svn_fs_fs__id_part_t *txn_id,
1326 apr_uint64_t node_id,
1327 apr_uint64_t copy_id,
1331 char buffer[2 * SVN_INT64_BUFFER_SIZE + 2];
1334 p += svn__ui64tobase36(p, node_id);
1336 p += svn__ui64tobase36(p, copy_id);
1340 SVN_ERR(svn_io_file_open(&file,
1341 path_txn_next_ids(fs, txn_id, pool),
1342 APR_WRITE | APR_TRUNCATE,
1343 APR_OS_DEFAULT, pool));
1344 SVN_ERR(svn_io_file_write_full(file, buffer, p - buffer, NULL, pool));
1345 return svn_io_file_close(file, pool);
1348 /* Find out what the next unique node-id and copy-id are for
1349 transaction TXN_ID in filesystem FS. Store the results in *NODE_ID
1350 and *COPY_ID. The next node-id is used both for creating new unique
1351 nodes for the given transaction, as well as uniquifying representations.
1352 Perform all allocations in POOL. */
1353 static svn_error_t *
1354 read_next_ids(apr_uint64_t *node_id,
1355 apr_uint64_t *copy_id,
1357 const svn_fs_fs__id_part_t *txn_id,
1360 svn_stringbuf_t *buf;
1362 SVN_ERR(svn_fs_fs__read_content(&buf,
1363 path_txn_next_ids(fs, txn_id, pool),
1366 /* Parse this into two separate strings. */
1369 *node_id = svn__base36toui64(&str, str);
1371 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
1372 _("next-id file corrupt"));
1375 *copy_id = svn__base36toui64(&str, str);
1377 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
1378 _("next-id file corrupt"));
1380 return SVN_NO_ERROR;
1383 /* Get a new and unique to this transaction node-id for transaction
1384 TXN_ID in filesystem FS. Store the new node-id in *NODE_ID_P.
1385 Node-ids are guaranteed to be unique to this transction, but may
1386 not necessarily be sequential. Perform all allocations in POOL. */
1387 static svn_error_t *
1388 get_new_txn_node_id(svn_fs_fs__id_part_t *node_id_p,
1390 const svn_fs_fs__id_part_t *txn_id,
1393 apr_uint64_t node_id, copy_id;
1395 /* First read in the current next-ids file. */
1396 SVN_ERR(read_next_ids(&node_id, ©_id, fs, txn_id, pool));
1398 node_id_p->revision = SVN_INVALID_REVNUM;
1399 node_id_p->number = node_id;
1401 SVN_ERR(write_next_ids(fs, txn_id, ++node_id, copy_id, pool));
1403 return SVN_NO_ERROR;
1407 svn_fs_fs__reserve_copy_id(svn_fs_fs__id_part_t *copy_id_p,
1409 const svn_fs_fs__id_part_t *txn_id,
1412 apr_uint64_t node_id, copy_id;
1414 /* First read in the current next-ids file. */
1415 SVN_ERR(read_next_ids(&node_id, ©_id, fs, txn_id, pool));
1417 /* this is an in-txn ID now */
1418 copy_id_p->revision = SVN_INVALID_REVNUM;
1419 copy_id_p->number = copy_id;
1421 /* Update the ID counter file */
1422 SVN_ERR(write_next_ids(fs, txn_id, node_id, ++copy_id, pool));
1424 return SVN_NO_ERROR;
1428 svn_fs_fs__create_node(const svn_fs_id_t **id_p,
1430 node_revision_t *noderev,
1431 const svn_fs_fs__id_part_t *copy_id,
1432 const svn_fs_fs__id_part_t *txn_id,
1435 svn_fs_fs__id_part_t node_id;
1436 const svn_fs_id_t *id;
1438 /* Get a new node-id for this node. */
1439 SVN_ERR(get_new_txn_node_id(&node_id, fs, txn_id, pool));
1441 id = svn_fs_fs__id_txn_create(&node_id, copy_id, txn_id, pool);
1445 SVN_ERR(svn_fs_fs__put_node_revision(fs, noderev->id, noderev, FALSE, pool));
1449 return SVN_NO_ERROR;
1453 svn_fs_fs__purge_txn(svn_fs_t *fs,
1454 const char *txn_id_str,
1457 fs_fs_data_t *ffd = fs->fsap_data;
1458 svn_fs_fs__id_part_t txn_id;
1459 SVN_ERR(svn_fs_fs__id_txn_parse(&txn_id, txn_id_str));
1461 /* Remove the shared transaction object associated with this transaction. */
1462 SVN_ERR(purge_shared_txn(fs, &txn_id, pool));
1463 /* Remove the directory associated with this transaction. */
1464 SVN_ERR(svn_io_remove_dir2(svn_fs_fs__path_txn_dir(fs, &txn_id, pool),
1465 FALSE, NULL, NULL, pool));
1466 if (ffd->format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT)
1468 /* Delete protorev and its lock, which aren't in the txn
1469 directory. It's OK if they don't exist (for example, if this
1470 is post-commit and the proto-rev has been moved into
1472 SVN_ERR(svn_io_remove_file2(
1473 svn_fs_fs__path_txn_proto_rev(fs, &txn_id, pool),
1475 SVN_ERR(svn_io_remove_file2(
1476 svn_fs_fs__path_txn_proto_rev_lock(fs, &txn_id, pool),
1479 return SVN_NO_ERROR;
1484 svn_fs_fs__abort_txn(svn_fs_txn_t *txn,
1487 SVN_ERR(svn_fs__check_fs(txn->fs, TRUE));
1489 /* Now, purge the transaction. */
1490 SVN_ERR_W(svn_fs_fs__purge_txn(txn->fs, txn->id, pool),
1491 apr_psprintf(pool, _("Transaction '%s' cleanup failed"),
1494 return SVN_NO_ERROR;
1497 /* Assign the UNIQUIFIER member of REP based on the current state of TXN_ID
1498 * in FS. Allocate the uniquifier in POOL.
1500 static svn_error_t *
1501 set_uniquifier(svn_fs_t *fs,
1502 representation_t *rep,
1505 svn_fs_fs__id_part_t temp;
1506 fs_fs_data_t *ffd = fs->fsap_data;
1508 if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT)
1510 SVN_ERR(get_new_txn_node_id(&temp, fs, &rep->txn_id, pool));
1511 rep->uniquifier.noderev_txn_id = rep->txn_id;
1512 rep->uniquifier.number = temp.number;
1515 return SVN_NO_ERROR;
1518 /* Return TRUE if the TXN_ID member of REP is in use.
1520 static svn_boolean_t
1521 is_txn_rep(const representation_t *rep)
1523 return svn_fs_fs__id_txn_used(&rep->txn_id);
1526 /* Mark the TXN_ID member of REP as "unused".
1529 reset_txn_in_rep(representation_t *rep)
1531 svn_fs_fs__id_txn_reset(&rep->txn_id);
1535 svn_fs_fs__set_entry(svn_fs_t *fs,
1536 const svn_fs_fs__id_part_t *txn_id,
1537 node_revision_t *parent_noderev,
1539 const svn_fs_id_t *id,
1540 svn_node_kind_t kind,
1543 representation_t *rep = parent_noderev->data_rep;
1544 const char *filename
1545 = svn_fs_fs__path_txn_node_children(fs, parent_noderev->id, pool);
1548 svn_filesize_t filesize;
1549 fs_fs_data_t *ffd = fs->fsap_data;
1550 apr_pool_t *subpool = svn_pool_create(pool);
1552 if (!rep || !is_txn_rep(rep))
1554 apr_array_header_t *entries;
1556 /* Before we can modify the directory, we need to dump its old
1557 contents into a mutable representation file. */
1558 SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, parent_noderev,
1560 SVN_ERR(svn_io_file_open(&file, filename,
1561 APR_WRITE | APR_CREATE | APR_BUFFERED,
1562 APR_OS_DEFAULT, pool));
1563 out = svn_stream_from_aprfile2(file, TRUE, pool);
1564 SVN_ERR(unparse_dir_entries(entries, out, subpool));
1566 /* Mark the node-rev's data rep as mutable. */
1567 rep = apr_pcalloc(pool, sizeof(*rep));
1568 rep->revision = SVN_INVALID_REVNUM;
1569 rep->txn_id = *txn_id;
1570 SVN_ERR(set_uniquifier(fs, rep, pool));
1571 parent_noderev->data_rep = rep;
1572 SVN_ERR(svn_fs_fs__put_node_revision(fs, parent_noderev->id,
1573 parent_noderev, FALSE, pool));
1575 /* Immediately populate the txn dir cache to avoid re-reading
1576 * the file we just wrote. */
1577 if (ffd->txn_dir_cache)
1580 = svn_fs_fs__id_unparse(parent_noderev->id, subpool)->data;
1581 svn_fs_fs__dir_data_t dir_data;
1583 /* Flush APR buffers. */
1584 SVN_ERR(svn_io_file_flush(file, subpool));
1586 /* Obtain final file size to update txn_dir_cache. */
1587 SVN_ERR(svn_io_file_size_get(&filesize, file, subpool));
1589 /* Store in the cache. */
1590 dir_data.entries = entries;
1591 dir_data.txn_filesize = filesize;
1592 SVN_ERR(svn_cache__set(ffd->txn_dir_cache, key, &dir_data,
1596 svn_pool_clear(subpool);
1600 /* The directory rep is already mutable, so just open it for append. */
1601 SVN_ERR(svn_io_file_open(&file, filename, APR_WRITE | APR_APPEND,
1602 APR_OS_DEFAULT, subpool));
1603 out = svn_stream_from_aprfile2(file, TRUE, subpool);
1605 /* If the cache contents is stale, drop it.
1607 * Note that the directory file is append-only, i.e. if the size
1608 * did not change, the contents didn't either. */
1609 if (ffd->txn_dir_cache)
1612 = svn_fs_fs__id_unparse(parent_noderev->id, subpool)->data;
1613 svn_boolean_t found;
1614 svn_filesize_t cached_filesize;
1616 /* Get the file size that corresponds to the cached contents
1618 SVN_ERR(svn_cache__get_partial((void **)&cached_filesize, &found,
1619 ffd->txn_dir_cache, key,
1620 svn_fs_fs__extract_dir_filesize,
1623 /* File size info still matches?
1624 * If not, we need to drop the cache entry. */
1627 SVN_ERR(svn_io_file_size_get(&filesize, file, subpool));
1629 if (cached_filesize != filesize)
1630 SVN_ERR(svn_cache__set(ffd->txn_dir_cache, key, NULL,
1636 /* Append an incremental hash entry for the entry change. */
1639 svn_fs_dirent_t entry;
1644 SVN_ERR(unparse_dir_entry(&entry, out, subpool));
1648 SVN_ERR(svn_stream_printf(out, subpool, "D %" APR_SIZE_T_FMT "\n%s\n",
1649 strlen(name), name));
1652 /* Flush APR buffers. */
1653 SVN_ERR(svn_io_file_flush(file, subpool));
1655 /* Obtain final file size to update txn_dir_cache. */
1656 SVN_ERR(svn_io_file_size_get(&filesize, file, subpool));
1659 SVN_ERR(svn_io_file_close(file, subpool));
1660 svn_pool_clear(subpool);
1662 /* if we have a directory cache for this transaction, update it */
1663 if (ffd->txn_dir_cache)
1665 /* build parameters: name, new entry, new file size */
1667 svn_fs_fs__id_unparse(parent_noderev->id, subpool)->data;
1668 replace_baton_t baton;
1671 baton.new_entry = NULL;
1672 baton.txn_filesize = filesize;
1676 baton.new_entry = apr_pcalloc(subpool, sizeof(*baton.new_entry));
1677 baton.new_entry->name = name;
1678 baton.new_entry->kind = kind;
1679 baton.new_entry->id = id;
1682 /* actually update the cached directory (if cached) */
1683 SVN_ERR(svn_cache__set_partial(ffd->txn_dir_cache, key,
1684 svn_fs_fs__replace_dir_entry, &baton,
1688 svn_pool_destroy(subpool);
1689 return SVN_NO_ERROR;
1693 svn_fs_fs__add_change(svn_fs_t *fs,
1694 const svn_fs_fs__id_part_t *txn_id,
1696 const svn_fs_id_t *id,
1697 svn_fs_path_change_kind_t change_kind,
1698 svn_boolean_t text_mod,
1699 svn_boolean_t prop_mod,
1700 svn_boolean_t mergeinfo_mod,
1701 svn_node_kind_t node_kind,
1702 svn_revnum_t copyfrom_rev,
1703 const char *copyfrom_path,
1707 svn_fs_path_change2_t *change;
1708 apr_hash_t *changes = apr_hash_make(pool);
1710 /* Not using APR_BUFFERED to append change in one atomic write operation. */
1711 SVN_ERR(svn_io_file_open(&file, path_txn_changes(fs, txn_id, pool),
1712 APR_APPEND | APR_WRITE | APR_CREATE,
1713 APR_OS_DEFAULT, pool));
1715 change = svn_fs__path_change_create_internal(id, change_kind, pool);
1716 change->text_mod = text_mod;
1717 change->prop_mod = prop_mod;
1718 change->mergeinfo_mod = mergeinfo_mod
1720 : svn_tristate_false;
1721 change->node_kind = node_kind;
1722 change->copyfrom_known = TRUE;
1723 change->copyfrom_rev = copyfrom_rev;
1725 change->copyfrom_path = apr_pstrdup(pool, copyfrom_path);
1727 svn_hash_sets(changes, path, change);
1728 SVN_ERR(svn_fs_fs__write_changes(svn_stream_from_aprfile2(file, TRUE, pool),
1729 fs, changes, FALSE, pool));
1731 return svn_io_file_close(file, pool);
1734 /* Store the (ITEM_INDEX, OFFSET) pair in the txn's log-to-phys proto
1736 * Use POOL for allocations.
1737 * This function assumes that transaction TXN_ID in FS uses logical
1740 static svn_error_t *
1741 store_l2p_index_entry(svn_fs_t *fs,
1742 const svn_fs_fs__id_part_t *txn_id,
1744 apr_uint64_t item_index,
1750 SVN_ERR_ASSERT(svn_fs_fs__use_log_addressing(fs));
1752 path = svn_fs_fs__path_l2p_proto_index(fs, txn_id, pool);
1753 SVN_ERR(svn_fs_fs__l2p_proto_index_open(&file, path, pool));
1754 SVN_ERR(svn_fs_fs__l2p_proto_index_add_entry(file, offset,
1756 SVN_ERR(svn_io_file_close(file, pool));
1758 return SVN_NO_ERROR;
1761 /* Store ENTRY in the phys-to-log proto index file of transaction TXN_ID.
1762 * Use POOL for allocations.
1763 * This function assumes that transaction TXN_ID in FS uses logical
1766 static svn_error_t *
1767 store_p2l_index_entry(svn_fs_t *fs,
1768 const svn_fs_fs__id_part_t *txn_id,
1769 const svn_fs_fs__p2l_entry_t *entry,
1775 SVN_ERR_ASSERT(svn_fs_fs__use_log_addressing(fs));
1777 path = svn_fs_fs__path_p2l_proto_index(fs, txn_id, pool);
1778 SVN_ERR(svn_fs_fs__p2l_proto_index_open(&file, path, pool));
1779 SVN_ERR(svn_fs_fs__p2l_proto_index_add_entry(file, entry, pool));
1780 SVN_ERR(svn_io_file_close(file, pool));
1782 return SVN_NO_ERROR;
1785 /* Allocate an item index for the given MY_OFFSET in the transaction TXN_ID
1786 * of file system FS and return it in *ITEM_INDEX. For old formats, it
1787 * will simply return the offset as item index; in new formats, it will
1788 * increment the txn's item index counter file and store the mapping in
1789 * the proto index file. Use POOL for allocations.
1791 static svn_error_t *
1792 allocate_item_index(apr_uint64_t *item_index,
1794 const svn_fs_fs__id_part_t *txn_id,
1795 apr_off_t my_offset,
1798 if (svn_fs_fs__use_log_addressing(fs))
1801 char buffer[SVN_INT64_BUFFER_SIZE] = { 0 };
1802 svn_boolean_t eof = FALSE;
1803 apr_size_t to_write;
1804 apr_size_t bytes_read;
1805 apr_off_t offset = 0;
1807 /* read number, increment it and write it back to disk */
1808 SVN_ERR(svn_io_file_open(&file,
1809 svn_fs_fs__path_txn_item_index(fs, txn_id, pool),
1810 APR_READ | APR_WRITE | APR_CREATE,
1811 APR_OS_DEFAULT, pool));
1812 SVN_ERR(svn_io_file_read_full2(file, buffer, sizeof(buffer)-1,
1813 &bytes_read, &eof, pool));
1815 /* Item index file should be shorter than SVN_INT64_BUFFER_SIZE,
1816 otherwise we truncate data. */
1818 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
1819 _("Unexpected itemidx file length"));
1820 else if (bytes_read)
1821 SVN_ERR(svn_cstring_atoui64(item_index, buffer));
1823 *item_index = SVN_FS_FS__ITEM_INDEX_FIRST_USER;
1825 to_write = svn__ui64toa(buffer, *item_index + 1);
1826 SVN_ERR(svn_io_file_seek(file, APR_SET, &offset, pool));
1827 SVN_ERR(svn_io_file_write_full(file, buffer, to_write, NULL, pool));
1828 SVN_ERR(svn_io_file_close(file, pool));
1830 /* write log-to-phys index */
1831 SVN_ERR(store_l2p_index_entry(fs, txn_id, my_offset, *item_index, pool));
1835 *item_index = (apr_uint64_t)my_offset;
1838 return SVN_NO_ERROR;
1841 /* Baton used by fnv1a_write_handler to calculate the FNV checksum
1842 * before passing the data on to the INNER_STREAM.
1844 typedef struct fnv1a_stream_baton_t
1846 svn_stream_t *inner_stream;
1847 svn_checksum_ctx_t *context;
1848 } fnv1a_stream_baton_t;
1850 /* Implement svn_write_fn_t.
1851 * Update checksum and pass data on to inner stream.
1853 static svn_error_t *
1854 fnv1a_write_handler(void *baton,
1858 fnv1a_stream_baton_t *b = baton;
1860 SVN_ERR(svn_checksum_update(b->context, data, *len));
1861 SVN_ERR(svn_stream_write(b->inner_stream, data, len));
1863 return SVN_NO_ERROR;
1866 /* Return a stream that calculates a FNV checksum in *CONTEXT
1867 * over all data written to the stream and passes that data on
1868 * to INNER_STREAM. Allocate objects in POOL.
1870 static svn_stream_t *
1871 fnv1a_wrap_stream(svn_checksum_ctx_t **context,
1872 svn_stream_t *inner_stream,
1875 svn_stream_t *outer_stream;
1877 fnv1a_stream_baton_t *baton = apr_pcalloc(pool, sizeof(*baton));
1878 baton->inner_stream = inner_stream;
1879 baton->context = svn_checksum_ctx_create(svn_checksum_fnv1a_32x4, pool);
1880 *context = baton->context;
1882 outer_stream = svn_stream_create(baton, pool);
1883 svn_stream_set_write(outer_stream, fnv1a_write_handler);
1885 return outer_stream;
1888 /* Set *DIGEST to the FNV checksum calculated in CONTEXT.
1889 * Use SCRATCH_POOL for temporary allocations.
1891 static svn_error_t *
1892 fnv1a_checksum_finalize(apr_uint32_t *digest,
1893 svn_checksum_ctx_t *context,
1894 apr_pool_t *scratch_pool)
1896 svn_checksum_t *checksum;
1898 SVN_ERR(svn_checksum_final(&checksum, context, scratch_pool));
1899 SVN_ERR_ASSERT(checksum->kind == svn_checksum_fnv1a_32x4);
1900 *digest = ntohl(*(const apr_uint32_t *)(checksum->digest));
1902 return SVN_NO_ERROR;
1905 /* This baton is used by the representation writing streams. It keeps
1906 track of the checksum information as well as the total size of the
1907 representation so far. */
1908 struct rep_write_baton
1910 /* The FS we are writing to. */
1913 /* Actual file to which we are writing. */
1914 svn_stream_t *rep_stream;
1916 /* A stream from the delta combiner. Data written here gets
1917 deltified, then eventually written to rep_stream. */
1918 svn_stream_t *delta_stream;
1920 /* Where is this representation header stored. */
1921 apr_off_t rep_offset;
1923 /* Start of the actual data. */
1924 apr_off_t delta_start;
1926 /* How many bytes have been written to this rep already. */
1927 svn_filesize_t rep_size;
1929 /* The node revision for which we're writing out info. */
1930 node_revision_t *noderev;
1932 /* Actual output file. */
1934 /* Lock 'cookie' used to unlock the output file once we've finished
1938 svn_checksum_ctx_t *md5_checksum_ctx;
1939 svn_checksum_ctx_t *sha1_checksum_ctx;
1941 /* calculate a modified FNV-1a checksum of the on-disk representation */
1942 svn_checksum_ctx_t *fnv1a_checksum_ctx;
1944 /* Local / scratch pool, available for temporary allocations. */
1945 apr_pool_t *scratch_pool;
1947 /* Outer / result pool. */
1948 apr_pool_t *result_pool;
1951 /* Handler for the write method of the representation writable stream.
1952 BATON is a rep_write_baton, DATA is the data to write, and *LEN is
1953 the length of this data. */
1954 static svn_error_t *
1955 rep_write_contents(void *baton,
1959 struct rep_write_baton *b = baton;
1961 SVN_ERR(svn_checksum_update(b->md5_checksum_ctx, data, *len));
1962 SVN_ERR(svn_checksum_update(b->sha1_checksum_ctx, data, *len));
1963 b->rep_size += *len;
1965 /* If we are writing a delta, use that stream. */
1966 if (b->delta_stream)
1967 return svn_stream_write(b->delta_stream, data, len);
1969 return svn_stream_write(b->rep_stream, data, len);
1972 /* Set *SPANNED to the number of shards touched when walking WALK steps on
1973 * NODEREV's predecessor chain in FS. Use POOL for temporary allocations.
1975 static svn_error_t *
1976 shards_spanned(int *spanned,
1978 node_revision_t *noderev,
1982 fs_fs_data_t *ffd = fs->fsap_data;
1983 int shard_size = ffd->max_files_per_dir ? ffd->max_files_per_dir : 1;
1984 apr_pool_t *iterpool;
1986 int count = walk ? 1 : 0; /* The start of a walk already touches a shard. */
1987 svn_revnum_t shard, last_shard = ffd->youngest_rev_cache / shard_size;
1988 iterpool = svn_pool_create(pool);
1989 while (walk-- && noderev->predecessor_count)
1991 svn_pool_clear(iterpool);
1992 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs,
1993 noderev->predecessor_id, pool,
1995 shard = svn_fs_fs__id_rev(noderev->id) / shard_size;
1996 if (shard != last_shard)
2002 svn_pool_destroy(iterpool);
2005 return SVN_NO_ERROR;
2008 /* Given a node-revision NODEREV in filesystem FS, return the
2009 representation in *REP to use as the base for a text representation
2010 delta if PROPS is FALSE. If PROPS has been set, a suitable props
2011 base representation will be returned. Perform temporary allocations
2013 static svn_error_t *
2014 choose_delta_base(representation_t **rep,
2016 node_revision_t *noderev,
2017 svn_boolean_t props,
2020 /* The zero-based index (counting from the "oldest" end), along NODEREVs line
2021 * predecessors, of the node-rev we will use as delta base. */
2023 /* The length of the linear part of a delta chain. (Delta chains use
2024 * skip-delta bits for the high-order bits and are linear in the low-order
2027 node_revision_t *base;
2028 fs_fs_data_t *ffd = fs->fsap_data;
2029 apr_pool_t *iterpool;
2031 /* If we have no predecessors, or that one is empty, then use the empty
2032 * stream as a base. */
2033 if (! noderev->predecessor_count)
2036 return SVN_NO_ERROR;
2039 /* Flip the rightmost '1' bit of the predecessor count to determine
2040 which file rev (counting from 0) we want to use. (To see why
2041 count & (count - 1) unsets the rightmost set bit, think about how
2042 you decrement a binary number.) */
2043 count = noderev->predecessor_count;
2044 count = count & (count - 1);
2046 /* Finding the delta base over a very long distance can become extremely
2047 expensive for very deep histories, possibly causing client timeouts etc.
2048 OTOH, this is a rare operation and its gains are minimal. Lets simply
2049 start deltification anew close every other 1000 changes or so. */
2050 walk = noderev->predecessor_count - count;
2051 if (walk > (int)ffd->max_deltification_walk)
2054 return SVN_NO_ERROR;
2057 /* We use skip delta for limiting the number of delta operations
2058 along very long node histories. Close to HEAD however, we create
2059 a linear history to minimize delta size. */
2060 if (walk < (int)ffd->max_linear_deltification)
2063 SVN_ERR(shards_spanned(&shards, fs, noderev, walk, pool));
2065 /* We also don't want the linear deltification to span more shards
2066 than if deltas we used in a simple skip-delta scheme. */
2067 if ((1 << (--shards)) <= walk)
2068 count = noderev->predecessor_count - 1;
2071 /* Walk back a number of predecessors equal to the difference
2072 between count and the original predecessor count. (For example,
2073 if noderev has ten predecessors and we want the eighth file rev,
2074 walk back two predecessors.) */
2076 iterpool = svn_pool_create(pool);
2077 while ((count++) < noderev->predecessor_count)
2079 svn_pool_clear(iterpool);
2080 SVN_ERR(svn_fs_fs__get_node_revision(&base, fs,
2081 base->predecessor_id, pool,
2084 svn_pool_destroy(iterpool);
2086 /* return a suitable base representation */
2087 *rep = props ? base->prop_rep : base->data_rep;
2089 /* if we encountered a shared rep, its parent chain may be different
2090 * from the node-rev parent chain. */
2093 int chain_length = 0;
2094 int shard_count = 0;
2096 /* Very short rep bases are simply not worth it as we are unlikely
2097 * to re-coup the deltification space overhead of 20+ bytes. */
2098 svn_filesize_t rep_size = (*rep)->expanded_size;
2102 return SVN_NO_ERROR;
2105 /* Check whether the length of the deltification chain is acceptable.
2106 * Otherwise, shared reps may form a non-skipping delta chain in
2108 SVN_ERR(svn_fs_fs__rep_chain_length(&chain_length, &shard_count,
2111 /* Some reasonable limit, depending on how acceptable longer linear
2112 * chains are in this repo. Also, allow for some minimal chain. */
2113 if (chain_length >= 2 * (int)ffd->max_linear_deltification + 2)
2116 /* To make it worth opening additional shards / pack files, we
2117 * require that the reps have a certain minimal size. To deltify
2118 * against a rep in different shard, the lower limit is 512 bytes
2119 * and doubles with every extra shard to visit along the delta
2121 if ( shard_count > 1
2122 && ((svn_filesize_t)128 << shard_count) >= rep_size)
2126 return SVN_NO_ERROR;
2129 /* Something went wrong and the pool for the rep write is being
2130 cleared before we've finished writing the rep. So we need
2131 to remove the rep from the protorevfile and we need to unlock
2132 the protorevfile. */
2134 rep_write_cleanup(void *data)
2136 struct rep_write_baton *b = data;
2139 /* Truncate and close the protorevfile. */
2140 err = svn_io_file_trunc(b->file, b->rep_offset, b->scratch_pool);
2141 err = svn_error_compose_create(err, svn_io_file_close(b->file,
2144 /* Remove our lock regardless of any preceding errors so that the
2145 being_written flag is always removed and stays consistent with the
2146 file lock which will be removed no matter what since the pool is
2148 err = svn_error_compose_create(err,
2149 unlock_proto_rev(b->fs,
2150 svn_fs_fs__id_txn_id(b->noderev->id),
2151 b->lockcookie, b->scratch_pool));
2154 apr_status_t rc = err->apr_err;
2155 svn_error_clear(err);
2163 txdelta_to_svndiff(svn_txdelta_window_handler_t *handler,
2164 void **handler_baton,
2165 svn_stream_t *output,
2169 fs_fs_data_t *ffd = fs->fsap_data;
2170 int svndiff_version;
2172 if (ffd->delta_compression_type == compression_type_lz4)
2174 SVN_ERR_ASSERT_NO_RETURN(ffd->format >= SVN_FS_FS__MIN_SVNDIFF2_FORMAT);
2175 svndiff_version = 2;
2177 else if (ffd->delta_compression_type == compression_type_zlib)
2179 SVN_ERR_ASSERT_NO_RETURN(ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT);
2180 svndiff_version = 1;
2184 svndiff_version = 0;
2187 svn_txdelta_to_svndiff3(handler, handler_baton, output, svndiff_version,
2188 ffd->delta_compression_level, pool);
2191 /* Get a rep_write_baton and store it in *WB_P for the representation
2192 indicated by NODEREV in filesystem FS. Perform allocations in
2193 POOL. Only appropriate for file contents, not for props or
2194 directory contents. */
2195 static svn_error_t *
2196 rep_write_get_baton(struct rep_write_baton **wb_p,
2198 node_revision_t *noderev,
2201 struct rep_write_baton *b;
2203 representation_t *base_rep;
2204 svn_stream_t *source;
2205 svn_txdelta_window_handler_t wh;
2207 svn_fs_fs__rep_header_t header = { 0 };
2209 b = apr_pcalloc(pool, sizeof(*b));
2211 b->sha1_checksum_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool);
2212 b->md5_checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool);
2215 b->result_pool = pool;
2216 b->scratch_pool = svn_pool_create(pool);
2218 b->noderev = noderev;
2220 /* Open the prototype rev file and seek to its end. */
2221 SVN_ERR(get_writable_proto_rev(&file, &b->lockcookie,
2222 fs, svn_fs_fs__id_txn_id(noderev->id),
2226 b->rep_stream = svn_stream_from_aprfile2(file, TRUE, b->scratch_pool);
2227 if (svn_fs_fs__use_log_addressing(fs))
2228 b->rep_stream = fnv1a_wrap_stream(&b->fnv1a_checksum_ctx, b->rep_stream,
2231 SVN_ERR(svn_io_file_get_offset(&b->rep_offset, file, b->scratch_pool));
2233 /* Get the base for this delta. */
2234 SVN_ERR(choose_delta_base(&base_rep, fs, noderev, FALSE, b->scratch_pool));
2235 SVN_ERR(svn_fs_fs__get_contents(&source, fs, base_rep, TRUE,
2238 /* Write out the rep header. */
2241 header.base_revision = base_rep->revision;
2242 header.base_item_index = base_rep->item_index;
2243 header.base_length = base_rep->size;
2244 header.type = svn_fs_fs__rep_delta;
2248 header.type = svn_fs_fs__rep_self_delta;
2250 SVN_ERR(svn_fs_fs__write_rep_header(&header, b->rep_stream,
2253 /* Now determine the offset of the actual svndiff data. */
2254 SVN_ERR(svn_io_file_get_offset(&b->delta_start, file,
2257 /* Cleanup in case something goes wrong. */
2258 apr_pool_cleanup_register(b->scratch_pool, b, rep_write_cleanup,
2259 apr_pool_cleanup_null);
2261 /* Prepare to write the svndiff data. */
2262 txdelta_to_svndiff(&wh, &whb, b->rep_stream, fs, pool);
2264 b->delta_stream = svn_txdelta_target_push(wh, whb, source,
2269 return SVN_NO_ERROR;
2272 /* For REP->SHA1_CHECKSUM, try to find an already existing representation
2273 in FS and return it in *OLD_REP. If no such representation exists or
2274 if rep sharing has been disabled for FS, NULL will be returned. Since
2275 there may be new duplicate representations within the same uncommitted
2276 revision, those can be passed in REPS_HASH (maps a sha1 digest onto
2277 representation_t*), otherwise pass in NULL for REPS_HASH.
2279 The content of both representations will be compared, taking REP's content
2280 from FILE at OFFSET. Only if they actually match, will *OLD_REP not be
2283 Use RESULT_POOL for *OLD_REP allocations and SCRATCH_POOL for temporaries.
2284 The lifetime of *OLD_REP is limited by both, RESULT_POOL and REP lifetime.
2286 static svn_error_t *
2287 get_shared_rep(representation_t **old_rep,
2289 representation_t *rep,
2292 apr_hash_t *reps_hash,
2293 apr_pool_t *result_pool,
2294 apr_pool_t *scratch_pool)
2297 fs_fs_data_t *ffd = fs->fsap_data;
2299 svn_checksum_t checksum;
2300 checksum.digest = rep->sha1_digest;
2301 checksum.kind = svn_checksum_sha1;
2303 /* Return NULL, if rep sharing has been disabled. */
2305 if (!ffd->rep_sharing_allowed)
2306 return SVN_NO_ERROR;
2308 /* Can't look up if we don't know the key (happens for directories). */
2310 return SVN_NO_ERROR;
2312 /* Check and see if we already have a representation somewhere that's
2313 identical to the one we just wrote out. Start with the hash lookup
2314 because it is cheapest. */
2316 *old_rep = apr_hash_get(reps_hash,
2318 APR_SHA1_DIGESTSIZE);
2320 /* If we haven't found anything yet, try harder and consult our DB. */
2321 if (*old_rep == NULL)
2323 err = svn_fs_fs__get_rep_reference(old_rep, fs, &checksum, result_pool);
2324 /* ### Other error codes that we shouldn't mask out? */
2325 if (err == SVN_NO_ERROR)
2328 SVN_ERR(svn_fs_fs__check_rep(*old_rep, fs, NULL, scratch_pool));
2330 else if (err->apr_err == SVN_ERR_FS_CORRUPT
2331 || SVN_ERROR_IN_CATEGORY(err->apr_err,
2332 SVN_ERR_MALFUNC_CATEGORY_START))
2334 /* Fatal error; don't mask it.
2336 In particular, this block is triggered when the rep-cache refers
2337 to revisions in the future. We signal that as a corruption situation
2338 since, once those revisions are less than youngest (because of more
2339 commits), the rep-cache would be invalid.
2345 /* Something's wrong with the rep-sharing index. We can continue
2346 without rep-sharing, but warn.
2348 (fs->warning)(fs->warning_baton, err);
2349 svn_error_clear(err);
2354 /* look for intra-revision matches (usually data reps but not limited
2355 to them in case props happen to look like some data rep)
2357 if (*old_rep == NULL && is_txn_rep(rep))
2359 svn_node_kind_t kind;
2360 const char *file_name
2361 = path_txn_sha1(fs, &rep->txn_id, rep->sha1_digest, scratch_pool);
2363 /* in our txn, is there a rep file named with the wanted SHA1?
2364 If so, read it and use that rep.
2366 SVN_ERR(svn_io_check_path(file_name, &kind, scratch_pool));
2367 if (kind == svn_node_file)
2369 svn_stringbuf_t *rep_string;
2370 SVN_ERR(svn_stringbuf_from_file2(&rep_string, file_name,
2372 SVN_ERR(svn_fs_fs__parse_representation(old_rep, rep_string,
2373 result_pool, scratch_pool));
2378 return SVN_NO_ERROR;
2380 /* A simple guard against general rep-cache induced corruption. */
2381 if ((*old_rep)->expanded_size != rep->expanded_size)
2383 /* Make the problem show up in the server log.
2385 Because not sharing reps is always a safe option,
2386 terminating the request would be inappropriate.
2388 err = svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2389 "Rep size %s mismatches rep-cache.db value %s "
2391 "You should delete the rep-cache.db and "
2392 "verify the repository. The cached rep will "
2394 apr_psprintf(scratch_pool,
2395 "%" SVN_FILESIZE_T_FMT,
2396 rep->expanded_size),
2397 apr_psprintf(scratch_pool,
2398 "%" SVN_FILESIZE_T_FMT,
2399 (*old_rep)->expanded_size),
2400 svn_checksum_to_cstring_display(&checksum,
2403 (fs->warning)(fs->warning_baton, err);
2404 svn_error_clear(err);
2406 /* Ignore the shared rep. */
2411 /* Add information that is missing in the cached data.
2412 Use the old rep for this content. */
2413 memcpy((*old_rep)->md5_digest, rep->md5_digest, sizeof(rep->md5_digest));
2414 (*old_rep)->uniquifier = rep->uniquifier;
2417 /* If we (very likely) found a matching representation, compare the actual
2418 * contents such that we can be sure that no rep-cache.db corruption or
2419 * hash collision produced a false positive. */
2422 apr_off_t old_position;
2423 svn_stream_t *contents;
2424 svn_stream_t *old_contents;
2427 /* The existing representation may itself be part of the current
2428 * transaction. In that case, it may be in different stages of
2429 * the commit finalization process.
2431 * OLD_REP_NORM is the same as that OLD_REP but it is assigned
2432 * explicitly to REP's transaction if OLD_REP does not point
2433 * to an already committed revision. This then prevents the
2434 * revision lookup and the txn data will be accessed.
2436 representation_t old_rep_norm = **old_rep;
2437 if ( !SVN_IS_VALID_REVNUM(old_rep_norm.revision)
2438 || old_rep_norm.revision > ffd->youngest_rev_cache)
2439 old_rep_norm.txn_id = rep->txn_id;
2441 /* Make sure we can later restore FILE's current position. */
2442 SVN_ERR(svn_io_file_get_offset(&old_position, file, scratch_pool));
2444 /* Compare the two representations.
2445 * Note that the stream comparison might also produce MD5 checksum
2446 * errors or other failures in case of SHA1 collisions. */
2447 SVN_ERR(svn_fs_fs__get_contents_from_file(&contents, fs, rep, file,
2448 offset, scratch_pool));
2449 SVN_ERR(svn_fs_fs__get_contents(&old_contents, fs, &old_rep_norm,
2450 FALSE, scratch_pool));
2451 err = svn_stream_contents_same2(&same, contents, old_contents,
2454 /* A mismatch should be extremely rare.
2455 * If it does happen, reject the commit. */
2458 /* SHA1 collision or worse. */
2459 svn_stringbuf_t *old_rep_str
2460 = svn_fs_fs__unparse_representation(*old_rep,
2464 svn_stringbuf_t *rep_str
2465 = svn_fs_fs__unparse_representation(rep,
2469 const char *checksum__str
2470 = svn_checksum_to_cstring_display(&checksum, scratch_pool);
2472 return svn_error_createf(SVN_ERR_FS_AMBIGUOUS_CHECKSUM_REP,
2473 err, "SHA1 of reps '%s' and '%s' "
2474 "matches (%s) but contents differ",
2475 old_rep_str->data, rep_str->data,
2479 /* Restore FILE's read / write position. */
2480 SVN_ERR(svn_io_file_seek(file, APR_SET, &old_position, scratch_pool));
2483 return SVN_NO_ERROR;
2486 /* Copy the hash sum calculation results from MD5_CTX, SHA1_CTX into REP.
2487 * SHA1 results are only be set if SHA1_CTX is not NULL.
2488 * Use POOL for allocations.
2490 static svn_error_t *
2491 digests_final(representation_t *rep,
2492 const svn_checksum_ctx_t *md5_ctx,
2493 const svn_checksum_ctx_t *sha1_ctx,
2496 svn_checksum_t *checksum;
2498 SVN_ERR(svn_checksum_final(&checksum, md5_ctx, pool));
2499 memcpy(rep->md5_digest, checksum->digest, svn_checksum_size(checksum));
2500 rep->has_sha1 = sha1_ctx != NULL;
2503 SVN_ERR(svn_checksum_final(&checksum, sha1_ctx, pool));
2504 memcpy(rep->sha1_digest, checksum->digest, svn_checksum_size(checksum));
2507 return SVN_NO_ERROR;
2510 /* Close handler for the representation write stream. BATON is a
2511 rep_write_baton. Writes out a new node-rev that correctly
2512 references the representation we just finished writing. */
2513 static svn_error_t *
2514 rep_write_contents_close(void *baton)
2516 struct rep_write_baton *b = baton;
2517 representation_t *rep;
2518 representation_t *old_rep;
2521 rep = apr_pcalloc(b->result_pool, sizeof(*rep));
2523 /* Close our delta stream so the last bits of svndiff are written
2525 if (b->delta_stream)
2526 SVN_ERR(svn_stream_close(b->delta_stream));
2528 /* Determine the length of the svndiff data. */
2529 SVN_ERR(svn_io_file_get_offset(&offset, b->file, b->scratch_pool));
2530 rep->size = offset - b->delta_start;
2532 /* Fill in the rest of the representation field. */
2533 rep->expanded_size = b->rep_size;
2534 rep->txn_id = *svn_fs_fs__id_txn_id(b->noderev->id);
2535 SVN_ERR(set_uniquifier(b->fs, rep, b->scratch_pool));
2536 rep->revision = SVN_INVALID_REVNUM;
2538 /* Finalize the checksum. */
2539 SVN_ERR(digests_final(rep, b->md5_checksum_ctx, b->sha1_checksum_ctx,
2542 /* Check and see if we already have a representation somewhere that's
2543 identical to the one we just wrote out. */
2544 SVN_ERR(get_shared_rep(&old_rep, b->fs, rep, b->file, b->rep_offset, NULL,
2545 b->result_pool, b->scratch_pool));
2549 /* We need to erase from the protorev the data we just wrote. */
2550 SVN_ERR(svn_io_file_trunc(b->file, b->rep_offset, b->scratch_pool));
2552 /* Use the old rep for this content. */
2553 b->noderev->data_rep = old_rep;
2557 /* Write out our cosmetic end marker. */
2558 SVN_ERR(svn_stream_puts(b->rep_stream, "ENDREP\n"));
2559 SVN_ERR(allocate_item_index(&rep->item_index, b->fs, &rep->txn_id,
2560 b->rep_offset, b->scratch_pool));
2562 b->noderev->data_rep = rep;
2565 /* Remove cleanup callback. */
2566 apr_pool_cleanup_kill(b->scratch_pool, b, rep_write_cleanup);
2568 /* Write out the new node-rev information. */
2569 SVN_ERR(svn_fs_fs__put_node_revision(b->fs, b->noderev->id, b->noderev,
2570 FALSE, b->scratch_pool));
2571 if (!old_rep && svn_fs_fs__use_log_addressing(b->fs))
2573 svn_fs_fs__p2l_entry_t entry;
2575 entry.offset = b->rep_offset;
2576 SVN_ERR(svn_io_file_get_offset(&offset, b->file, b->scratch_pool));
2577 entry.size = offset - b->rep_offset;
2578 entry.type = SVN_FS_FS__ITEM_TYPE_FILE_REP;
2579 entry.item.revision = SVN_INVALID_REVNUM;
2580 entry.item.number = rep->item_index;
2581 SVN_ERR(fnv1a_checksum_finalize(&entry.fnv1_checksum,
2582 b->fnv1a_checksum_ctx,
2585 SVN_ERR(store_p2l_index_entry(b->fs, &rep->txn_id, &entry,
2589 SVN_ERR(svn_io_file_close(b->file, b->scratch_pool));
2591 /* Write the sha1->rep mapping *after* we successfully written node
2592 * revision to disk. */
2594 SVN_ERR(store_sha1_rep_mapping(b->fs, b->noderev, b->scratch_pool));
2596 SVN_ERR(unlock_proto_rev(b->fs, &rep->txn_id, b->lockcookie,
2598 svn_pool_destroy(b->scratch_pool);
2600 return SVN_NO_ERROR;
2603 /* Store a writable stream in *CONTENTS_P that will receive all data
2604 written and store it as the file data representation referenced by
2605 NODEREV in filesystem FS. Perform temporary allocations in
2606 POOL. Only appropriate for file data, not props or directory
2608 static svn_error_t *
2609 set_representation(svn_stream_t **contents_p,
2611 node_revision_t *noderev,
2614 struct rep_write_baton *wb;
2616 if (! svn_fs_fs__id_is_txn(noderev->id))
2617 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
2618 _("Attempted to write to non-transaction '%s'"),
2619 svn_fs_fs__id_unparse(noderev->id, pool)->data);
2621 SVN_ERR(rep_write_get_baton(&wb, fs, noderev, pool));
2623 *contents_p = svn_stream_create(wb, pool);
2624 svn_stream_set_write(*contents_p, rep_write_contents);
2625 svn_stream_set_close(*contents_p, rep_write_contents_close);
2627 return SVN_NO_ERROR;
2631 svn_fs_fs__set_contents(svn_stream_t **stream,
2633 node_revision_t *noderev,
2636 if (noderev->kind != svn_node_file)
2637 return svn_error_create(SVN_ERR_FS_NOT_FILE, NULL,
2638 _("Can't set text contents of a directory"));
2640 return set_representation(stream, fs, noderev, pool);
2644 svn_fs_fs__create_successor(const svn_fs_id_t **new_id_p,
2646 const svn_fs_id_t *old_idp,
2647 node_revision_t *new_noderev,
2648 const svn_fs_fs__id_part_t *copy_id,
2649 const svn_fs_fs__id_part_t *txn_id,
2652 const svn_fs_id_t *id;
2655 copy_id = svn_fs_fs__id_copy_id(old_idp);
2656 id = svn_fs_fs__id_txn_create(svn_fs_fs__id_node_id(old_idp), copy_id,
2659 new_noderev->id = id;
2661 if (! new_noderev->copyroot_path)
2663 new_noderev->copyroot_path = apr_pstrdup(pool,
2664 new_noderev->created_path);
2665 new_noderev->copyroot_rev = svn_fs_fs__id_rev(new_noderev->id);
2668 SVN_ERR(svn_fs_fs__put_node_revision(fs, new_noderev->id, new_noderev, FALSE,
2673 return SVN_NO_ERROR;
2677 svn_fs_fs__set_proplist(svn_fs_t *fs,
2678 node_revision_t *noderev,
2679 apr_hash_t *proplist,
2682 const char *filename
2683 = svn_fs_fs__path_txn_node_props(fs, noderev->id, pool);
2687 /* Dump the property list to the mutable property file. */
2688 SVN_ERR(svn_io_file_open(&file, filename,
2689 APR_WRITE | APR_CREATE | APR_TRUNCATE
2690 | APR_BUFFERED, APR_OS_DEFAULT, pool));
2691 out = svn_stream_from_aprfile2(file, TRUE, pool);
2692 SVN_ERR(svn_hash_write2(proplist, out, SVN_HASH_TERMINATOR, pool));
2693 SVN_ERR(svn_io_file_close(file, pool));
2695 /* Mark the node-rev's prop rep as mutable, if not already done. */
2696 if (!noderev->prop_rep || !is_txn_rep(noderev->prop_rep))
2698 noderev->prop_rep = apr_pcalloc(pool, sizeof(*noderev->prop_rep));
2699 noderev->prop_rep->txn_id = *svn_fs_fs__id_txn_id(noderev->id);
2700 SVN_ERR(set_uniquifier(fs, noderev->prop_rep, pool));
2701 noderev->prop_rep->revision = SVN_INVALID_REVNUM;
2702 SVN_ERR(svn_fs_fs__put_node_revision(fs, noderev->id, noderev, FALSE,
2706 return SVN_NO_ERROR;
2709 /* This baton is used by the stream created for write_container_rep. */
2710 struct write_container_baton
2712 svn_stream_t *stream;
2716 svn_checksum_ctx_t *md5_ctx;
2718 /* SHA1 calculation is optional. If not needed, this will be NULL. */
2719 svn_checksum_ctx_t *sha1_ctx;
2722 /* The handler for the write_container_rep stream. BATON is a
2723 write_container_baton, DATA has the data to write and *LEN is the number
2724 of bytes to write. */
2725 static svn_error_t *
2726 write_container_handler(void *baton,
2730 struct write_container_baton *whb = baton;
2732 SVN_ERR(svn_checksum_update(whb->md5_ctx, data, *len));
2734 SVN_ERR(svn_checksum_update(whb->sha1_ctx, data, *len));
2736 SVN_ERR(svn_stream_write(whb->stream, data, len));
2739 return SVN_NO_ERROR;
2742 /* Callback function type. Write the data provided by BATON into STREAM. */
2743 typedef svn_error_t *
2744 (* collection_writer_t)(svn_stream_t *stream, void *baton, apr_pool_t *pool);
2746 /* Implement collection_writer_t writing the C string->svn_string_t hash
2748 static svn_error_t *
2749 write_hash_to_stream(svn_stream_t *stream,
2753 apr_hash_t *hash = baton;
2754 SVN_ERR(svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, pool));
2756 return SVN_NO_ERROR;
2759 /* Implement collection_writer_t writing the svn_fs_dirent_t* array given
2761 static svn_error_t *
2762 write_directory_to_stream(svn_stream_t *stream,
2766 apr_array_header_t *dir = baton;
2767 SVN_ERR(unparse_dir_entries(dir, stream, pool));
2769 return SVN_NO_ERROR;
2772 /* Write out the COLLECTION as a text representation to file FILE using
2773 WRITER. In the process, record position, the total size of the dump and
2774 MD5 as well as SHA1 in REP. Add the representation of type ITEM_TYPE to
2775 the indexes if necessary.
2777 If ALLOW_REP_SHARING is FALSE, rep-sharing will not be used, regardless
2778 of any other option and rep-sharing settings. If rep sharing has been
2779 enabled and REPS_HASH is not NULL, it will be used in addition to the
2780 on-disk cache to find earlier reps with the same content. If such
2781 existing reps can be found, we will truncate the one just written from
2782 the file and return the existing rep.
2784 Perform temporary allocations in SCRATCH_POOL. */
2785 static svn_error_t *
2786 write_container_rep(representation_t *rep,
2789 collection_writer_t writer,
2791 apr_hash_t *reps_hash,
2792 svn_boolean_t allow_rep_sharing,
2793 apr_uint32_t item_type,
2794 apr_pool_t *scratch_pool)
2796 svn_stream_t *stream;
2797 struct write_container_baton *whb;
2798 svn_checksum_ctx_t *fnv1a_checksum_ctx;
2799 apr_off_t offset = 0;
2801 SVN_ERR(svn_io_file_get_offset(&offset, file, scratch_pool));
2803 whb = apr_pcalloc(scratch_pool, sizeof(*whb));
2805 whb->stream = svn_stream_from_aprfile2(file, TRUE, scratch_pool);
2806 if (svn_fs_fs__use_log_addressing(fs))
2807 whb->stream = fnv1a_wrap_stream(&fnv1a_checksum_ctx, whb->stream,
2810 fnv1a_checksum_ctx = NULL;
2812 whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool);
2813 if (item_type != SVN_FS_FS__ITEM_TYPE_DIR_REP)
2814 whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, scratch_pool);
2816 stream = svn_stream_create(whb, scratch_pool);
2817 svn_stream_set_write(stream, write_container_handler);
2819 SVN_ERR(svn_stream_puts(whb->stream, "PLAIN\n"));
2821 SVN_ERR(writer(stream, collection, scratch_pool));
2823 /* Store the results. */
2824 SVN_ERR(digests_final(rep, whb->md5_ctx, whb->sha1_ctx, scratch_pool));
2826 /* Update size info. */
2827 rep->expanded_size = whb->size;
2828 rep->size = whb->size;
2830 /* Check and see if we already have a representation somewhere that's
2831 identical to the one we just wrote out. */
2832 if (allow_rep_sharing)
2834 representation_t *old_rep;
2835 SVN_ERR(get_shared_rep(&old_rep, fs, rep, file, offset, reps_hash,
2836 scratch_pool, scratch_pool));
2840 /* We need to erase from the protorev the data we just wrote. */
2841 SVN_ERR(svn_io_file_trunc(file, offset, scratch_pool));
2843 /* Use the old rep for this content. */
2844 memcpy(rep, old_rep, sizeof (*rep));
2845 return SVN_NO_ERROR;
2849 /* Write out our cosmetic end marker. */
2850 SVN_ERR(svn_stream_puts(whb->stream, "ENDREP\n"));
2852 SVN_ERR(allocate_item_index(&rep->item_index, fs, &rep->txn_id,
2853 offset, scratch_pool));
2855 if (svn_fs_fs__use_log_addressing(fs))
2857 svn_fs_fs__p2l_entry_t entry;
2859 entry.offset = offset;
2860 SVN_ERR(svn_io_file_get_offset(&offset, file, scratch_pool));
2861 entry.size = offset - entry.offset;
2862 entry.type = item_type;
2863 entry.item.revision = SVN_INVALID_REVNUM;
2864 entry.item.number = rep->item_index;
2865 SVN_ERR(fnv1a_checksum_finalize(&entry.fnv1_checksum,
2869 SVN_ERR(store_p2l_index_entry(fs, &rep->txn_id, &entry, scratch_pool));
2872 return SVN_NO_ERROR;
2875 /* Write out the COLLECTION pertaining to the NODEREV in FS as a deltified
2876 text representation to file FILE using WRITER. In the process, record the
2877 total size and the md5 digest in REP and add the representation of type
2878 ITEM_TYPE to the indexes if necessary.
2880 If ALLOW_REP_SHARING is FALSE, rep-sharing will not be used, regardless
2881 of any other option and rep-sharing settings. If rep sharing has been
2882 enabled and REPS_HASH is not NULL, it will be used in addition to the
2883 on-disk cache to find earlier reps with the same content. If such
2884 existing reps can be found, we will truncate the one just written from
2885 the file and return the existing rep.
2887 If ITEM_TYPE is IS_PROPS equals SVN_FS_FS__ITEM_TYPE_*_PROPS, assume
2888 that we want to a props representation as the base for our delta.
2889 Perform temporary allocations in SCRATCH_POOL.
2891 static svn_error_t *
2892 write_container_delta_rep(representation_t *rep,
2895 collection_writer_t writer,
2897 node_revision_t *noderev,
2898 apr_hash_t *reps_hash,
2899 svn_boolean_t allow_rep_sharing,
2900 apr_uint32_t item_type,
2901 apr_pool_t *scratch_pool)
2903 svn_txdelta_window_handler_t diff_wh;
2906 svn_stream_t *file_stream;
2907 svn_stream_t *stream;
2908 representation_t *base_rep;
2909 svn_checksum_ctx_t *fnv1a_checksum_ctx;
2910 svn_stream_t *source;
2911 svn_fs_fs__rep_header_t header = { 0 };
2913 apr_off_t rep_end = 0;
2914 apr_off_t delta_start = 0;
2915 apr_off_t offset = 0;
2917 struct write_container_baton *whb;
2918 svn_boolean_t is_props = (item_type == SVN_FS_FS__ITEM_TYPE_FILE_PROPS)
2919 || (item_type == SVN_FS_FS__ITEM_TYPE_DIR_PROPS);
2921 /* Get the base for this delta. */
2922 SVN_ERR(choose_delta_base(&base_rep, fs, noderev, is_props, scratch_pool));
2923 SVN_ERR(svn_fs_fs__get_contents(&source, fs, base_rep, FALSE, scratch_pool));
2925 SVN_ERR(svn_io_file_get_offset(&offset, file, scratch_pool));
2927 /* Write out the rep header. */
2930 header.base_revision = base_rep->revision;
2931 header.base_item_index = base_rep->item_index;
2932 header.base_length = base_rep->size;
2933 header.type = svn_fs_fs__rep_delta;
2937 header.type = svn_fs_fs__rep_self_delta;
2940 file_stream = svn_stream_from_aprfile2(file, TRUE, scratch_pool);
2941 if (svn_fs_fs__use_log_addressing(fs))
2942 file_stream = fnv1a_wrap_stream(&fnv1a_checksum_ctx, file_stream,
2945 fnv1a_checksum_ctx = NULL;
2946 SVN_ERR(svn_fs_fs__write_rep_header(&header, file_stream, scratch_pool));
2947 SVN_ERR(svn_io_file_get_offset(&delta_start, file, scratch_pool));
2949 /* Prepare to write the svndiff data. */
2950 txdelta_to_svndiff(&diff_wh, &diff_whb, file_stream, fs, scratch_pool);
2952 whb = apr_pcalloc(scratch_pool, sizeof(*whb));
2953 whb->stream = svn_txdelta_target_push(diff_wh, diff_whb, source,
2956 whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool);
2957 if (item_type != SVN_FS_FS__ITEM_TYPE_DIR_REP)
2958 whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, scratch_pool);
2960 /* serialize the hash */
2961 stream = svn_stream_create(whb, scratch_pool);
2962 svn_stream_set_write(stream, write_container_handler);
2964 SVN_ERR(writer(stream, collection, scratch_pool));
2965 SVN_ERR(svn_stream_close(whb->stream));
2967 /* Store the results. */
2968 SVN_ERR(digests_final(rep, whb->md5_ctx, whb->sha1_ctx, scratch_pool));
2970 /* Update size info. */
2971 SVN_ERR(svn_io_file_get_offset(&rep_end, file, scratch_pool));
2972 rep->size = rep_end - delta_start;
2973 rep->expanded_size = whb->size;
2975 /* Check and see if we already have a representation somewhere that's
2976 identical to the one we just wrote out. */
2977 if (allow_rep_sharing)
2979 representation_t *old_rep;
2980 SVN_ERR(get_shared_rep(&old_rep, fs, rep, file, offset, reps_hash,
2981 scratch_pool, scratch_pool));
2985 /* We need to erase from the protorev the data we just wrote. */
2986 SVN_ERR(svn_io_file_trunc(file, offset, scratch_pool));
2988 /* Use the old rep for this content. */
2989 memcpy(rep, old_rep, sizeof (*rep));
2990 return SVN_NO_ERROR;
2994 /* Write out our cosmetic end marker. */
2995 SVN_ERR(svn_stream_puts(file_stream, "ENDREP\n"));
2997 SVN_ERR(allocate_item_index(&rep->item_index, fs, &rep->txn_id,
2998 offset, scratch_pool));
3000 if (svn_fs_fs__use_log_addressing(fs))
3002 svn_fs_fs__p2l_entry_t entry;
3004 entry.offset = offset;
3005 SVN_ERR(svn_io_file_get_offset(&offset, file, scratch_pool));
3006 entry.size = offset - entry.offset;
3007 entry.type = item_type;
3008 entry.item.revision = SVN_INVALID_REVNUM;
3009 entry.item.number = rep->item_index;
3010 SVN_ERR(fnv1a_checksum_finalize(&entry.fnv1_checksum,
3014 SVN_ERR(store_p2l_index_entry(fs, &rep->txn_id, &entry, scratch_pool));
3017 return SVN_NO_ERROR;
3020 /* Sanity check ROOT_NODEREV, a candidate for being the root node-revision
3021 of (not yet committed) revision REV in FS. Use POOL for temporary
3024 If you change this function, consider updating svn_fs_fs__verify() too.
3026 static svn_error_t *
3027 validate_root_noderev(svn_fs_t *fs,
3028 node_revision_t *root_noderev,
3032 svn_revnum_t head_revnum = rev-1;
3033 int head_predecessor_count;
3035 SVN_ERR_ASSERT(rev > 0);
3037 /* Compute HEAD_PREDECESSOR_COUNT. */
3039 svn_fs_root_t *head_revision;
3040 const svn_fs_id_t *head_root_id;
3041 node_revision_t *head_root_noderev;
3043 /* Get /@HEAD's noderev. */
3044 SVN_ERR(svn_fs_fs__revision_root(&head_revision, fs, head_revnum, pool));
3045 SVN_ERR(svn_fs_fs__node_id(&head_root_id, head_revision, "/", pool));
3046 SVN_ERR(svn_fs_fs__get_node_revision(&head_root_noderev, fs, head_root_id,
3048 head_predecessor_count = head_root_noderev->predecessor_count;
3051 /* Check that the root noderev's predecessor count equals REV.
3053 This kind of corruption was seen on svn.apache.org (both on
3054 the root noderev and on other fspaths' noderevs); see
3057 Normally (rev == root_noderev->predecessor_count), but here we
3058 use a more roundabout check that should only trigger on new instances
3059 of the corruption, rather than trigger on each and every new commit
3060 to a repository that has triggered the bug somewhere in its root
3063 if ( (root_noderev->predecessor_count - head_predecessor_count)
3064 != (rev - head_revnum))
3066 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
3067 _("predecessor count for "
3068 "the root node-revision is wrong: "
3069 "found (%d+%ld != %d), committing r%ld"),
3070 head_predecessor_count,
3071 rev - head_revnum, /* This is equal to 1. */
3072 root_noderev->predecessor_count,
3076 return SVN_NO_ERROR;
3079 /* Given the potentially txn-local id PART, update that to a permanent ID
3080 * based on the REVISION currently being written and the START_ID for that
3081 * revision. Use the repo FORMAT to decide which implementation to use.
3084 get_final_id(svn_fs_fs__id_part_t *part,
3085 svn_revnum_t revision,
3086 apr_uint64_t start_id,
3089 if (part->revision == SVN_INVALID_REVNUM)
3091 if (format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT)
3093 part->revision = revision;
3098 part->number += start_id;
3103 /* Copy a node-revision specified by id ID in fileystem FS from a
3104 transaction into the proto-rev-file FILE. Set *NEW_ID_P to a
3105 pointer to the new node-id which will be allocated in POOL.
3106 If this is a directory, copy all children as well.
3108 START_NODE_ID and START_COPY_ID are
3109 the first available node and copy ids for this filesystem, for older
3112 REV is the revision number that this proto-rev-file will represent.
3114 INITIAL_OFFSET is the offset of the proto-rev-file on entry to
3117 Collect the pair_cache_key_t of all directories written to the
3118 committed cache in DIRECTORY_IDS.
3120 If REPS_TO_CACHE is not NULL, append to it a copy (allocated in
3121 REPS_POOL) of each data rep that is new in this revision.
3123 If REPS_HASH is not NULL, append copies (allocated in REPS_POOL)
3124 of the representations of each property rep that is new in this
3127 AT_ROOT is true if the node revision being written is the root
3128 node-revision. It is only controls additional sanity checking
3131 Temporary allocations are also from POOL. */
3132 static svn_error_t *
3133 write_final_rev(const svn_fs_id_t **new_id_p,
3137 const svn_fs_id_t *id,
3138 apr_uint64_t start_node_id,
3139 apr_uint64_t start_copy_id,
3140 apr_off_t initial_offset,
3141 apr_array_header_t *directory_ids,
3142 apr_array_header_t *reps_to_cache,
3143 apr_hash_t *reps_hash,
3144 apr_pool_t *reps_pool,
3145 svn_boolean_t at_root,
3148 node_revision_t *noderev;
3149 apr_off_t my_offset;
3150 const svn_fs_id_t *new_id;
3151 svn_fs_fs__id_part_t node_id, copy_id, rev_item;
3152 fs_fs_data_t *ffd = fs->fsap_data;
3153 const svn_fs_fs__id_part_t *txn_id = svn_fs_fs__id_txn_id(id);
3154 svn_stream_t *file_stream;
3155 svn_checksum_ctx_t *fnv1a_checksum_ctx;
3156 apr_pool_t *subpool;
3160 /* Check to see if this is a transaction node. */
3161 if (! svn_fs_fs__id_is_txn(id))
3162 return SVN_NO_ERROR;
3164 subpool = svn_pool_create(pool);
3165 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, id, pool, subpool));
3167 if (noderev->kind == svn_node_dir)
3169 apr_array_header_t *entries;
3172 /* This is a directory. Write out all the children first. */
3174 SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, noderev, pool,
3176 for (i = 0; i < entries->nelts; ++i)
3178 svn_fs_dirent_t *dirent
3179 = APR_ARRAY_IDX(entries, i, svn_fs_dirent_t *);
3181 svn_pool_clear(subpool);
3182 SVN_ERR(write_final_rev(&new_id, file, rev, fs, dirent->id,
3183 start_node_id, start_copy_id, initial_offset,
3184 directory_ids, reps_to_cache, reps_hash,
3185 reps_pool, FALSE, subpool));
3186 if (new_id && (svn_fs_fs__id_rev(new_id) == rev))
3187 dirent->id = svn_fs_fs__id_copy(new_id, pool);
3190 if (noderev->data_rep && is_txn_rep(noderev->data_rep))
3192 pair_cache_key_t *key;
3193 svn_fs_fs__dir_data_t dir_data;
3195 /* Write out the contents of this directory as a text rep. */
3196 noderev->data_rep->revision = rev;
3197 if (ffd->deltify_directories)
3198 SVN_ERR(write_container_delta_rep(noderev->data_rep, file,
3200 write_directory_to_stream,
3201 fs, noderev, NULL, FALSE,
3202 SVN_FS_FS__ITEM_TYPE_DIR_REP,
3205 SVN_ERR(write_container_rep(noderev->data_rep, file, entries,
3206 write_directory_to_stream, fs, NULL,
3207 FALSE, SVN_FS_FS__ITEM_TYPE_DIR_REP,
3210 reset_txn_in_rep(noderev->data_rep);
3212 /* Cache the new directory contents. Otherwise, subsequent reads
3213 * or commits will likely have to reconstruct, verify and parse
3215 key = apr_array_push(directory_ids);
3216 key->revision = noderev->data_rep->revision;
3217 key->second = noderev->data_rep->item_index;
3219 /* Store directory contents under the new revision number but mark
3220 * it as "stale" by setting the file length to 0. Committed dirs
3221 * will report -1, in-txn dirs will report > 0, so that this can
3222 * never match. We reset that to -1 after the commit is complete.
3224 dir_data.entries = entries;
3225 dir_data.txn_filesize = 0;
3227 SVN_ERR(svn_cache__set(ffd->dir_cache, key, &dir_data, subpool));
3232 /* This is a file. We should make sure the data rep, if it
3233 exists in a "this" state, gets rewritten to our new revision
3236 if (noderev->data_rep && is_txn_rep(noderev->data_rep))
3238 reset_txn_in_rep(noderev->data_rep);
3239 noderev->data_rep->revision = rev;
3241 if (!svn_fs_fs__use_log_addressing(fs))
3243 /* See issue 3845. Some unknown mechanism caused the
3244 protorev file to get truncated, so check for that
3246 if (noderev->data_rep->item_index + noderev->data_rep->size
3248 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
3249 _("Truncated protorev file detected"));
3254 svn_pool_destroy(subpool);
3256 /* Fix up the property reps. */
3257 if (noderev->prop_rep && is_txn_rep(noderev->prop_rep))
3259 apr_hash_t *proplist;
3260 apr_uint32_t item_type = noderev->kind == svn_node_dir
3261 ? SVN_FS_FS__ITEM_TYPE_DIR_PROPS
3262 : SVN_FS_FS__ITEM_TYPE_FILE_PROPS;
3263 SVN_ERR(svn_fs_fs__get_proplist(&proplist, fs, noderev, pool));
3264 noderev->prop_rep->txn_id = *txn_id;
3265 SVN_ERR(set_uniquifier(fs, noderev->prop_rep, pool));
3266 noderev->prop_rep->revision = rev;
3268 if (ffd->deltify_properties)
3269 SVN_ERR(write_container_delta_rep(noderev->prop_rep, file, proplist,
3270 write_hash_to_stream, fs, noderev,
3271 reps_hash, TRUE, item_type, pool));
3273 SVN_ERR(write_container_rep(noderev->prop_rep, file, proplist,
3274 write_hash_to_stream, fs, reps_hash,
3275 TRUE, item_type, pool));
3277 reset_txn_in_rep(noderev->prop_rep);
3280 /* Convert our temporary ID into a permanent revision one. */
3281 node_id = *svn_fs_fs__id_node_id(noderev->id);
3282 get_final_id(&node_id, rev, start_node_id, ffd->format);
3283 copy_id = *svn_fs_fs__id_copy_id(noderev->id);
3284 get_final_id(©_id, rev, start_copy_id, ffd->format);
3286 if (noderev->copyroot_rev == SVN_INVALID_REVNUM)
3287 noderev->copyroot_rev = rev;
3289 /* root nodes have a fixed ID in log addressing mode */
3290 SVN_ERR(svn_io_file_get_offset(&my_offset, file, pool));
3291 if (svn_fs_fs__use_log_addressing(fs) && at_root)
3293 /* reference the root noderev from the log-to-phys index */
3294 rev_item.number = SVN_FS_FS__ITEM_INDEX_ROOT_NODE;
3295 SVN_ERR(store_l2p_index_entry(fs, txn_id, my_offset,
3296 rev_item.number, pool));
3299 SVN_ERR(allocate_item_index(&rev_item.number, fs, txn_id,
3302 rev_item.revision = rev;
3303 new_id = svn_fs_fs__id_rev_create(&node_id, ©_id, &rev_item, pool);
3305 noderev->id = new_id;
3307 if (ffd->rep_sharing_allowed)
3309 /* Save the data representation's hash in the rep cache. */
3310 if ( noderev->data_rep && noderev->kind == svn_node_file
3311 && noderev->data_rep->revision == rev)
3313 SVN_ERR_ASSERT(reps_to_cache && reps_pool);
3314 APR_ARRAY_PUSH(reps_to_cache, representation_t *)
3315 = svn_fs_fs__rep_copy(noderev->data_rep, reps_pool);
3318 if (noderev->prop_rep && noderev->prop_rep->revision == rev)
3320 /* Add new property reps to hash and on-disk cache. */
3321 representation_t *copy
3322 = svn_fs_fs__rep_copy(noderev->prop_rep, reps_pool);
3324 SVN_ERR_ASSERT(reps_to_cache && reps_pool);
3325 APR_ARRAY_PUSH(reps_to_cache, representation_t *) = copy;
3327 apr_hash_set(reps_hash,
3329 APR_SHA1_DIGESTSIZE,
3334 /* don't serialize SHA1 for dir content to disk (waste of space) */
3335 /* ### Could clients record bogus last-changed-revisions (issue #4700)? */
3336 if (noderev->data_rep && noderev->kind == svn_node_dir)
3337 noderev->data_rep->has_sha1 = FALSE;
3339 /* Compatibility: while we don't need to serialize SHA1 for props (it is
3340 not used), older formats can only have representation strings that either
3341 have both the SHA1 value *and* the uniquifier, or don't have them at all.
3342 For such formats, both values get written to the disk only if the SHA1
3345 We cannot omit the uniquifier, as doing so breaks svn_fs_props_changed()
3346 for properties with shared representations, see issues #4623 and #4700.
3347 Therefore, we skip writing SHA1, but only for the newer formats where
3348 this dependency is untied and we can write the uniquifier to the disk
3351 if (ffd->format >= SVN_FS_FS__MIN_REP_STRING_OPTIONAL_VALUES_FORMAT &&
3354 noderev->prop_rep->has_sha1 = FALSE;
3357 /* Workaround issue #4031: is-fresh-txn-root in revision files. */
3358 noderev->is_fresh_txn_root = FALSE;
3360 /* Write out our new node-revision. */
3362 SVN_ERR(validate_root_noderev(fs, noderev, rev, pool));
3364 file_stream = svn_stream_from_aprfile2(file, TRUE, pool);
3365 if (svn_fs_fs__use_log_addressing(fs))
3366 file_stream = fnv1a_wrap_stream(&fnv1a_checksum_ctx, file_stream, pool);
3368 fnv1a_checksum_ctx = NULL;
3370 SVN_ERR(svn_fs_fs__write_noderev(file_stream, noderev, ffd->format,
3371 svn_fs_fs__fs_supports_mergeinfo(fs),
3374 /* reference the root noderev from the log-to-phys index */
3375 if (svn_fs_fs__use_log_addressing(fs))
3377 svn_fs_fs__p2l_entry_t entry;
3378 rev_item.revision = SVN_INVALID_REVNUM;
3380 entry.offset = my_offset;
3381 SVN_ERR(svn_io_file_get_offset(&my_offset, file, pool));
3382 entry.size = my_offset - entry.offset;
3383 entry.type = SVN_FS_FS__ITEM_TYPE_NODEREV;
3384 entry.item = rev_item;
3385 SVN_ERR(fnv1a_checksum_finalize(&entry.fnv1_checksum,
3389 SVN_ERR(store_p2l_index_entry(fs, txn_id, &entry, pool));
3392 /* Return our ID that references the revision file. */
3393 *new_id_p = noderev->id;
3395 return SVN_NO_ERROR;
3398 /* Write the changed path info CHANGED_PATHS from transaction TXN_ID to the
3399 permanent rev-file FILE in filesystem FS. *OFFSET_P is set the to offset
3400 in the file of the beginning of this information. Perform temporary
3401 allocations in POOL. */
3402 static svn_error_t *
3403 write_final_changed_path_info(apr_off_t *offset_p,
3406 const svn_fs_fs__id_part_t *txn_id,
3407 apr_hash_t *changed_paths,
3411 svn_stream_t *stream;
3412 svn_checksum_ctx_t *fnv1a_checksum_ctx;
3414 SVN_ERR(svn_io_file_get_offset(&offset, file, pool));
3416 /* write to target file & calculate checksum if needed */
3417 stream = svn_stream_from_aprfile2(file, TRUE, pool);
3418 if (svn_fs_fs__use_log_addressing(fs))
3419 stream = fnv1a_wrap_stream(&fnv1a_checksum_ctx, stream, pool);
3421 fnv1a_checksum_ctx = NULL;
3423 SVN_ERR(svn_fs_fs__write_changes(stream, fs, changed_paths, TRUE, pool));
3427 /* reference changes from the indexes */
3428 if (svn_fs_fs__use_log_addressing(fs))
3430 svn_fs_fs__p2l_entry_t entry;
3432 entry.offset = offset;
3433 SVN_ERR(svn_io_file_get_offset(&offset, file, pool));
3434 entry.size = offset - entry.offset;
3435 entry.type = SVN_FS_FS__ITEM_TYPE_CHANGES;
3436 entry.item.revision = SVN_INVALID_REVNUM;
3437 entry.item.number = SVN_FS_FS__ITEM_INDEX_CHANGES;
3438 SVN_ERR(fnv1a_checksum_finalize(&entry.fnv1_checksum,
3442 SVN_ERR(store_p2l_index_entry(fs, txn_id, &entry, pool));
3443 SVN_ERR(store_l2p_index_entry(fs, txn_id, entry.offset,
3444 SVN_FS_FS__ITEM_INDEX_CHANGES, pool));
3447 return SVN_NO_ERROR;
3450 /* Open a new svn_fs_t handle to FS, set that handle's concept of "current
3451 youngest revision" to NEW_REV, and call svn_fs_fs__verify_root() on
3452 NEW_REV's revision root.
3454 Intended to be called as the very last step in a commit before 'current'
3455 is bumped. This implies that we are holding the write lock. */
3456 static svn_error_t *
3457 verify_before_commit(svn_fs_t *fs,
3458 svn_revnum_t new_rev,
3461 fs_fs_data_t *ffd = fs->fsap_data;
3462 svn_fs_t *ft; /* fs++ == ft */
3463 svn_fs_root_t *root;
3464 fs_fs_data_t *ft_ffd;
3465 apr_hash_t *fs_config;
3467 SVN_ERR_ASSERT(ffd->svn_fs_open_);
3469 /* make sure FT does not simply return data cached by other instances
3470 * but actually retrieves it from disk at least once.
3472 fs_config = apr_hash_make(pool);
3473 svn_hash_sets(fs_config, SVN_FS_CONFIG_FSFS_CACHE_NS,
3474 svn_uuid_generate(pool));
3475 SVN_ERR(ffd->svn_fs_open_(&ft, fs->path,
3479 ft_ffd = ft->fsap_data;
3480 /* Don't let FT consult rep-cache.db, either. */
3481 ft_ffd->rep_sharing_allowed = FALSE;
3484 ft_ffd->youngest_rev_cache = new_rev;
3486 SVN_ERR(svn_fs_fs__revision_root(&root, ft, new_rev, pool));
3487 SVN_ERR_ASSERT(root->is_txn_root == FALSE && root->rev == new_rev);
3488 SVN_ERR_ASSERT(ft_ffd->youngest_rev_cache == new_rev);
3489 SVN_ERR(svn_fs_fs__verify_root(root, pool));
3491 return SVN_NO_ERROR;
3494 /* Update the 'current' file to hold the correct next node and copy_ids
3495 from transaction TXN_ID in filesystem FS. The current revision is
3496 set to REV. Perform temporary allocations in POOL. */
3497 static svn_error_t *
3498 write_final_current(svn_fs_t *fs,
3499 const svn_fs_fs__id_part_t *txn_id,
3501 apr_uint64_t start_node_id,
3502 apr_uint64_t start_copy_id,
3505 apr_uint64_t txn_node_id;
3506 apr_uint64_t txn_copy_id;
3507 fs_fs_data_t *ffd = fs->fsap_data;
3509 if (ffd->format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT)
3510 return svn_fs_fs__write_current(fs, rev, 0, 0, pool);
3512 /* To find the next available ids, we add the id that used to be in
3513 the 'current' file, to the next ids from the transaction file. */
3514 SVN_ERR(read_next_ids(&txn_node_id, &txn_copy_id, fs, txn_id, pool));
3516 start_node_id += txn_node_id;
3517 start_copy_id += txn_copy_id;
3519 return svn_fs_fs__write_current(fs, rev, start_node_id, start_copy_id,
3523 /* Verify that the user registered with FS has all the locks necessary to
3524 permit all the changes associated with TXN_NAME.
3525 The FS write lock is assumed to be held by the caller. */
3526 static svn_error_t *
3527 verify_locks(svn_fs_t *fs,
3528 const svn_fs_fs__id_part_t *txn_id,
3529 apr_hash_t *changed_paths,
3532 apr_pool_t *iterpool;
3533 apr_array_header_t *changed_paths_sorted;
3534 svn_stringbuf_t *last_recursed = NULL;
3537 /* Make an array of the changed paths, and sort them depth-first-ily. */
3538 changed_paths_sorted = svn_sort__hash(changed_paths,
3539 svn_sort_compare_items_as_paths,
3542 /* Now, traverse the array of changed paths, verify locks. Note
3543 that if we need to do a recursive verification a path, we'll skip
3544 over children of that path when we get to them. */
3545 iterpool = svn_pool_create(pool);
3546 for (i = 0; i < changed_paths_sorted->nelts; i++)
3548 const svn_sort__item_t *item;
3550 svn_fs_path_change2_t *change;
3551 svn_boolean_t recurse = TRUE;
3553 svn_pool_clear(iterpool);
3555 item = &APR_ARRAY_IDX(changed_paths_sorted, i, svn_sort__item_t);
3557 /* Fetch the change associated with our path. */
3559 change = item->value;
3561 /* If this path has already been verified as part of a recursive
3562 check of one of its parents, no need to do it again. */
3564 && svn_fspath__skip_ancestor(last_recursed->data, path))
3567 /* What does it mean to succeed at lock verification for a given
3568 path? For an existing file or directory getting modified
3569 (text, props), it means we hold the lock on the file or
3570 directory. For paths being added or removed, we need to hold
3571 the locks for that path and any children of that path.
3573 WHEW! We have no reliable way to determine the node kind
3574 of deleted items, but fortunately we are going to do a
3575 recursive check on deleted paths regardless of their kind. */
3576 if (change->change_kind == svn_fs_path_change_modify)
3578 SVN_ERR(svn_fs_fs__allow_locked_operation(path, fs, recurse, TRUE,
3581 /* If we just did a recursive check, remember the path we
3582 checked (so children can be skipped). */
3585 if (! last_recursed)
3586 last_recursed = svn_stringbuf_create(path, pool);
3588 svn_stringbuf_set(last_recursed, path);
3591 svn_pool_destroy(iterpool);
3592 return SVN_NO_ERROR;
3595 /* Writes final revision properties to file PATH applying permissions
3596 from file PERMS_REFERENCE. This involves setting svn:date and
3597 removing any temporary properties associated with the commit flags. */
3598 static svn_error_t *
3599 write_final_revprop(const char *path,
3600 const char *perms_reference,
3602 svn_boolean_t flush_to_disk,
3605 apr_hash_t *txnprops;
3607 svn_string_t *client_date;
3608 apr_file_t *revprop_file;
3609 svn_stream_t *stream;
3611 SVN_ERR(svn_fs_fs__txn_proplist(&txnprops, txn, pool));
3613 /* Remove any temporary txn props representing 'flags'. */
3614 svn_hash_sets(txnprops, SVN_FS__PROP_TXN_CHECK_OOD, NULL);
3615 svn_hash_sets(txnprops, SVN_FS__PROP_TXN_CHECK_LOCKS, NULL);
3617 client_date = svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CLIENT_DATE);
3620 svn_hash_sets(txnprops, SVN_FS__PROP_TXN_CLIENT_DATE, NULL);
3623 /* Update commit time to ensure that svn:date revprops remain ordered if
3625 if (!client_date || strcmp(client_date->data, "1"))
3627 date.data = svn_time_to_cstring(apr_time_now(), pool);
3628 date.len = strlen(date.data);
3629 svn_hash_sets(txnprops, SVN_PROP_REVISION_DATE, &date);
3632 /* Create new revprops file. Tell OS to truncate existing file,
3633 since file may already exists from failed transaction. */
3634 SVN_ERR(svn_io_file_open(&revprop_file, path,
3635 APR_WRITE | APR_CREATE | APR_TRUNCATE
3636 | APR_BUFFERED, APR_OS_DEFAULT, pool));
3638 stream = svn_stream_from_aprfile2(revprop_file, TRUE, pool);
3639 SVN_ERR(svn_hash_write2(txnprops, stream, SVN_HASH_TERMINATOR, pool));
3640 SVN_ERR(svn_stream_close(stream));
3643 SVN_ERR(svn_io_file_flush_to_disk(revprop_file, pool));
3644 SVN_ERR(svn_io_file_close(revprop_file, pool));
3646 SVN_ERR(svn_io_copy_perms(perms_reference, path, pool));
3648 return SVN_NO_ERROR;
3652 svn_fs_fs__add_index_data(svn_fs_t *fs,
3654 const char *l2p_proto_index,
3655 const char *p2l_proto_index,
3656 svn_revnum_t revision,
3659 apr_off_t l2p_offset;
3660 apr_off_t p2l_offset;
3661 svn_stringbuf_t *footer;
3662 unsigned char footer_length;
3663 svn_checksum_t *l2p_checksum;
3664 svn_checksum_t *p2l_checksum;
3666 /* Append the actual index data to the pack file. */
3668 SVN_ERR(svn_io_file_seek(file, APR_END, &l2p_offset, pool));
3669 SVN_ERR(svn_fs_fs__l2p_index_append(&l2p_checksum, fs, file,
3670 l2p_proto_index, revision,
3674 SVN_ERR(svn_io_file_seek(file, APR_END, &p2l_offset, pool));
3675 SVN_ERR(svn_fs_fs__p2l_index_append(&p2l_checksum, fs, file,
3676 p2l_proto_index, revision,
3679 /* Append footer. */
3680 footer = svn_fs_fs__unparse_footer(l2p_offset, l2p_checksum,
3681 p2l_offset, p2l_checksum, pool, pool);
3682 SVN_ERR(svn_io_file_write_full(file, footer->data, footer->len, NULL,
3685 footer_length = footer->len;
3686 SVN_ERR_ASSERT(footer_length == footer->len);
3687 SVN_ERR(svn_io_file_write_full(file, &footer_length, 1, NULL, pool));
3689 return SVN_NO_ERROR;
3692 /* Mark the directories cached in FS with the keys from DIRECTORY_IDS
3693 * as "valid" now. Use SCRATCH_POOL for temporaries. */
3694 static svn_error_t *
3695 promote_cached_directories(svn_fs_t *fs,
3696 apr_array_header_t *directory_ids,
3697 apr_pool_t *scratch_pool)
3699 fs_fs_data_t *ffd = fs->fsap_data;
3700 apr_pool_t *iterpool;
3703 if (!ffd->dir_cache)
3704 return SVN_NO_ERROR;
3706 iterpool = svn_pool_create(scratch_pool);
3707 for (i = 0; i < directory_ids->nelts; ++i)
3709 const pair_cache_key_t *key
3710 = &APR_ARRAY_IDX(directory_ids, i, pair_cache_key_t);
3712 svn_pool_clear(iterpool);
3714 /* Currently, the entry for KEY - if it still exists - is marked
3715 * as "stale" and would not be used. Mark it as current for in-
3717 SVN_ERR(svn_cache__set_partial(ffd->dir_cache, key,
3718 svn_fs_fs__reset_txn_filesize, NULL,
3722 svn_pool_destroy(iterpool);
3724 return SVN_NO_ERROR;
3727 /* Baton used for commit_body below. */
3728 struct commit_baton {
3729 svn_revnum_t *new_rev_p;
3732 apr_array_header_t *reps_to_cache;
3733 apr_hash_t *reps_hash;
3734 apr_pool_t *reps_pool;
3737 /* The work-horse for svn_fs_fs__commit, called with the FS write lock.
3738 This implements the svn_fs_fs__with_write_lock() 'body' callback
3739 type. BATON is a 'struct commit_baton *'. */
3740 static svn_error_t *
3741 commit_body(void *baton, apr_pool_t *pool)
3743 struct commit_baton *cb = baton;
3744 fs_fs_data_t *ffd = cb->fs->fsap_data;
3745 const char *old_rev_filename, *rev_filename, *proto_filename;
3746 const char *revprop_filename;
3747 const svn_fs_id_t *root_id, *new_root_id;
3748 apr_uint64_t start_node_id;
3749 apr_uint64_t start_copy_id;
3750 svn_revnum_t old_rev, new_rev;
3751 apr_file_t *proto_file;
3752 void *proto_file_lockcookie;
3753 apr_off_t initial_offset, changed_path_offset;
3754 const svn_fs_fs__id_part_t *txn_id = svn_fs_fs__txn_get_id(cb->txn);
3755 apr_hash_t *changed_paths;
3756 apr_array_header_t *directory_ids = apr_array_make(pool, 4,
3757 sizeof(pair_cache_key_t));
3759 /* Re-Read the current repository format. All our repo upgrade and
3760 config evaluation strategies are such that existing information in
3761 FS and FFD remains valid.
3763 Although we don't recommend upgrading hot repositories, people may
3764 still do it and we must make sure to either handle them gracefully
3767 Committing pre-format 3 txns will fail after upgrade to format 3+
3768 because the proto-rev cannot be found; no further action needed.
3769 Upgrades from pre-f7 to f7+ means a potential change in addressing
3770 mode for the final rev. We must be sure to detect that cause because
3771 the failure would only manifest once the new revision got committed.
3773 SVN_ERR(svn_fs_fs__read_format_file(cb->fs, pool));
3775 /* Read the current youngest revision and, possibly, the next available
3776 node id and copy id (for old format filesystems). Update the cached
3777 value for the youngest revision, because we have just checked it. */
3778 SVN_ERR(svn_fs_fs__read_current(&old_rev, &start_node_id, &start_copy_id,
3780 ffd->youngest_rev_cache = old_rev;
3782 /* Check to make sure this transaction is based off the most recent
3784 if (cb->txn->base_rev != old_rev)
3785 return svn_error_create(SVN_ERR_FS_TXN_OUT_OF_DATE, NULL,
3786 _("Transaction out of date"));
3788 /* We need the changes list for verification as well as for writing it
3789 to the final rev file. */
3790 SVN_ERR(svn_fs_fs__txn_changes_fetch(&changed_paths, cb->fs, txn_id,
3793 /* Locks may have been added (or stolen) between the calling of
3794 previous svn_fs.h functions and svn_fs_commit_txn(), so we need
3795 to re-examine every changed-path in the txn and re-verify all
3796 discovered locks. */
3797 SVN_ERR(verify_locks(cb->fs, txn_id, changed_paths, pool));
3799 /* We are going to be one better than this puny old revision. */
3800 new_rev = old_rev + 1;
3802 /* Get a write handle on the proto revision file. */
3803 SVN_ERR(get_writable_proto_rev(&proto_file, &proto_file_lockcookie,
3804 cb->fs, txn_id, pool));
3805 SVN_ERR(svn_io_file_get_offset(&initial_offset, proto_file, pool));
3807 /* Write out all the node-revisions and directory contents. */
3808 root_id = svn_fs_fs__id_txn_create_root(txn_id, pool);
3809 SVN_ERR(write_final_rev(&new_root_id, proto_file, new_rev, cb->fs, root_id,
3810 start_node_id, start_copy_id, initial_offset,
3811 directory_ids, cb->reps_to_cache, cb->reps_hash,
3812 cb->reps_pool, TRUE, pool));
3814 /* Write the changed-path information. */
3815 SVN_ERR(write_final_changed_path_info(&changed_path_offset, proto_file,
3816 cb->fs, txn_id, changed_paths,
3819 if (svn_fs_fs__use_log_addressing(cb->fs))
3821 /* Append the index data to the rev file. */
3822 SVN_ERR(svn_fs_fs__add_index_data(cb->fs, proto_file,
3823 svn_fs_fs__path_l2p_proto_index(cb->fs, txn_id, pool),
3824 svn_fs_fs__path_p2l_proto_index(cb->fs, txn_id, pool),
3829 /* Write the final line. */
3831 svn_stringbuf_t *trailer
3832 = svn_fs_fs__unparse_revision_trailer
3833 ((apr_off_t)svn_fs_fs__id_item(new_root_id),
3834 changed_path_offset,
3836 SVN_ERR(svn_io_file_write_full(proto_file, trailer->data, trailer->len,
3840 if (ffd->flush_to_disk)
3841 SVN_ERR(svn_io_file_flush_to_disk(proto_file, pool));
3842 SVN_ERR(svn_io_file_close(proto_file, pool));
3844 /* We don't unlock the prototype revision file immediately to avoid a
3845 race with another caller writing to the prototype revision file
3846 before we commit it. */
3848 /* Create the shard for the rev and revprop file, if we're sharding and
3849 this is the first revision of a new shard. We don't care if this
3850 fails because the shard already existed for some reason. */
3851 if (ffd->max_files_per_dir && new_rev % ffd->max_files_per_dir == 0)
3853 /* Create the revs shard. */
3856 = svn_fs_fs__path_rev_shard(cb->fs, new_rev, pool);
3858 = svn_io_dir_make(new_dir, APR_OS_DEFAULT, pool);
3859 if (err && !APR_STATUS_IS_EEXIST(err->apr_err))
3860 return svn_error_trace(err);
3861 svn_error_clear(err);
3862 SVN_ERR(svn_io_copy_perms(svn_dirent_join(cb->fs->path,
3868 /* Create the revprops shard. */
3869 SVN_ERR_ASSERT(! svn_fs_fs__is_packed_revprop(cb->fs, new_rev));
3872 = svn_fs_fs__path_revprops_shard(cb->fs, new_rev, pool);
3874 = svn_io_dir_make(new_dir, APR_OS_DEFAULT, pool);
3875 if (err && !APR_STATUS_IS_EEXIST(err->apr_err))
3876 return svn_error_trace(err);
3877 svn_error_clear(err);
3878 SVN_ERR(svn_io_copy_perms(svn_dirent_join(cb->fs->path,
3885 /* Move the finished rev file into place.
3887 ### This "breaks" the transaction by removing the protorev file
3888 ### but the revision is not yet complete. If this commit does
3889 ### not complete for any reason the transaction will be lost. */
3890 old_rev_filename = svn_fs_fs__path_rev_absolute(cb->fs, old_rev, pool);
3891 rev_filename = svn_fs_fs__path_rev(cb->fs, new_rev, pool);
3892 proto_filename = svn_fs_fs__path_txn_proto_rev(cb->fs, txn_id, pool);
3893 SVN_ERR(svn_fs_fs__move_into_place(proto_filename, rev_filename,
3894 old_rev_filename, ffd->flush_to_disk,
3897 /* Now that we've moved the prototype revision file out of the way,
3898 we can unlock it (since further attempts to write to the file
3899 will fail as it no longer exists). We must do this so that we can
3900 remove the transaction directory later. */
3901 SVN_ERR(unlock_proto_rev(cb->fs, txn_id, proto_file_lockcookie, pool));
3903 /* Write final revprops file. */
3904 SVN_ERR_ASSERT(! svn_fs_fs__is_packed_revprop(cb->fs, new_rev));
3905 revprop_filename = svn_fs_fs__path_revprops(cb->fs, new_rev, pool);
3906 SVN_ERR(write_final_revprop(revprop_filename, old_rev_filename,
3907 cb->txn, ffd->flush_to_disk, pool));
3909 /* Run paranoia checks. */
3910 if (ffd->verify_before_commit)
3912 SVN_ERR(verify_before_commit(cb->fs, new_rev, pool));
3915 /* Update the 'current' file. */
3916 SVN_ERR(write_final_current(cb->fs, txn_id, new_rev, start_node_id,
3917 start_copy_id, pool));
3919 /* At this point the new revision is committed and globally visible
3920 so let the caller know it succeeded by giving it the new revision
3921 number, which fulfills svn_fs_commit_txn() contract. Any errors
3922 after this point do not change the fact that a new revision was
3924 *cb->new_rev_p = new_rev;
3926 ffd->youngest_rev_cache = new_rev;
3928 /* Make the directory contents alreday cached for the new revision
3930 SVN_ERR(promote_cached_directories(cb->fs, directory_ids, pool));
3932 /* Remove this transaction directory. */
3933 SVN_ERR(svn_fs_fs__purge_txn(cb->fs, cb->txn->id, pool));
3935 return SVN_NO_ERROR;
3938 /* Add the representations in REPS_TO_CACHE (an array of representation_t *)
3939 * to the rep-cache database of FS. */
3940 static svn_error_t *
3941 write_reps_to_cache(svn_fs_t *fs,
3942 const apr_array_header_t *reps_to_cache,
3943 apr_pool_t *scratch_pool)
3947 for (i = 0; i < reps_to_cache->nelts; i++)
3949 representation_t *rep = APR_ARRAY_IDX(reps_to_cache, i, representation_t *);
3951 SVN_ERR(svn_fs_fs__set_rep_reference(fs, rep, scratch_pool));
3954 return SVN_NO_ERROR;
3958 svn_fs_fs__commit(svn_revnum_t *new_rev_p,
3963 struct commit_baton cb;
3964 fs_fs_data_t *ffd = fs->fsap_data;
3966 cb.new_rev_p = new_rev_p;
3970 if (ffd->rep_sharing_allowed)
3972 cb.reps_to_cache = apr_array_make(pool, 5, sizeof(representation_t *));
3973 cb.reps_hash = apr_hash_make(pool);
3974 cb.reps_pool = pool;
3978 cb.reps_to_cache = NULL;
3979 cb.reps_hash = NULL;
3980 cb.reps_pool = NULL;
3983 SVN_ERR(svn_fs_fs__with_write_lock(fs, commit_body, &cb, pool));
3985 /* At this point, *NEW_REV_P has been set, so errors below won't affect
3986 the success of the commit. (See svn_fs_commit_txn().) */
3988 if (ffd->rep_sharing_allowed)
3992 SVN_ERR(svn_fs_fs__open_rep_cache(fs, pool));
3994 /* Write new entries to the rep-sharing database.
3996 * We use an sqlite transaction to speed things up;
3997 * see <http://www.sqlite.org/faq.html#q19>.
3999 /* ### A commit that touches thousands of files will starve other
4000 (reader/writer) commits for the duration of the below call.
4001 Maybe write in batches? */
4002 SVN_ERR(svn_sqlite__begin_transaction(ffd->rep_cache_db));
4003 err = write_reps_to_cache(fs, cb.reps_to_cache, pool);
4004 err = svn_sqlite__finish_transaction(ffd->rep_cache_db, err);
4006 if (svn_error_find_cause(err, SVN_ERR_SQLITE_ROLLBACK_FAILED))
4008 /* Failed rollback means that our db connection is unusable, and
4009 the only thing we can do is close it. The connection will be
4010 reopened during the next operation with rep-cache.db. */
4011 return svn_error_trace(
4012 svn_error_compose_create(err,
4013 svn_fs_fs__close_rep_cache(fs)));
4016 return svn_error_trace(err);
4019 return SVN_NO_ERROR;
4024 svn_fs_fs__list_transactions(apr_array_header_t **names_p,
4028 const char *txn_dir;
4029 apr_hash_t *dirents;
4030 apr_hash_index_t *hi;
4031 apr_array_header_t *names;
4032 apr_size_t ext_len = strlen(PATH_EXT_TXN);
4034 names = apr_array_make(pool, 1, sizeof(const char *));
4036 /* Get the transactions directory. */
4037 txn_dir = svn_fs_fs__path_txns_dir(fs, pool);
4039 /* Now find a listing of this directory. */
4040 SVN_ERR(svn_io_get_dirents3(&dirents, txn_dir, TRUE, pool, pool));
4042 /* Loop through all the entries and return anything that ends with '.txn'. */
4043 for (hi = apr_hash_first(pool, dirents); hi; hi = apr_hash_next(hi))
4045 const char *name = apr_hash_this_key(hi);
4046 apr_ssize_t klen = apr_hash_this_key_len(hi);
4049 /* The name must end with ".txn" to be considered a transaction. */
4050 if ((apr_size_t) klen <= ext_len
4051 || (strcmp(name + klen - ext_len, PATH_EXT_TXN)) != 0)
4054 /* Truncate the ".txn" extension and store the ID. */
4055 id = apr_pstrndup(pool, name, strlen(name) - ext_len);
4056 APR_ARRAY_PUSH(names, const char *) = id;
4061 return SVN_NO_ERROR;
4065 svn_fs_fs__open_txn(svn_fs_txn_t **txn_p,
4072 svn_node_kind_t kind;
4073 transaction_t *local_txn;
4074 svn_fs_fs__id_part_t txn_id;
4076 SVN_ERR(svn_fs_fs__id_txn_parse(&txn_id, name));
4078 /* First check to see if the directory exists. */
4079 SVN_ERR(svn_io_check_path(svn_fs_fs__path_txn_dir(fs, &txn_id, pool),
4082 /* Did we find it? */
4083 if (kind != svn_node_dir)
4084 return svn_error_createf(SVN_ERR_FS_NO_SUCH_TRANSACTION, NULL,
4085 _("No such transaction '%s'"),
4088 txn = apr_pcalloc(pool, sizeof(*txn));
4089 ftd = apr_pcalloc(pool, sizeof(*ftd));
4090 ftd->txn_id = txn_id;
4092 /* Read in the root node of this transaction. */
4093 txn->id = apr_pstrdup(pool, name);
4096 SVN_ERR(svn_fs_fs__get_txn(&local_txn, fs, &txn_id, pool));
4098 txn->base_rev = svn_fs_fs__id_rev(local_txn->base_id);
4100 txn->vtable = &txn_vtable;
4101 txn->fsap_data = ftd;
4104 return SVN_NO_ERROR;
4108 svn_fs_fs__txn_proplist(apr_hash_t **table_p,
4112 apr_hash_t *proplist = apr_hash_make(pool);
4113 SVN_ERR(get_txn_proplist(proplist, txn->fs, svn_fs_fs__txn_get_id(txn),
4115 *table_p = proplist;
4117 return SVN_NO_ERROR;
4122 svn_fs_fs__delete_node_revision(svn_fs_t *fs,
4123 const svn_fs_id_t *id,
4126 node_revision_t *noderev;
4128 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, id, pool, pool));
4130 /* Delete any mutable property representation. */
4131 if (noderev->prop_rep && is_txn_rep(noderev->prop_rep))
4132 SVN_ERR(svn_io_remove_file2(svn_fs_fs__path_txn_node_props(fs, id, pool),
4135 /* Delete any mutable data representation. */
4136 if (noderev->data_rep && is_txn_rep(noderev->data_rep)
4137 && noderev->kind == svn_node_dir)
4139 fs_fs_data_t *ffd = fs->fsap_data;
4140 SVN_ERR(svn_io_remove_file2(svn_fs_fs__path_txn_node_children(fs, id,
4144 /* remove the corresponding entry from the cache, if such exists */
4145 if (ffd->txn_dir_cache)
4147 const char *key = svn_fs_fs__id_unparse(id, pool)->data;
4148 SVN_ERR(svn_cache__set(ffd->txn_dir_cache, key, NULL, pool));
4152 return svn_io_remove_file2(svn_fs_fs__path_txn_node_rev(fs, id, pool),
4158 /*** Transactions ***/
4161 svn_fs_fs__get_txn_ids(const svn_fs_id_t **root_id_p,
4162 const svn_fs_id_t **base_root_id_p,
4164 const svn_fs_fs__id_part_t *txn_id,
4168 SVN_ERR(svn_fs_fs__get_txn(&txn, fs, txn_id, pool));
4169 *root_id_p = txn->root_id;
4170 *base_root_id_p = txn->base_id;
4171 return SVN_NO_ERROR;
4175 /* Generic transaction operations. */
4178 svn_fs_fs__txn_prop(svn_string_t **value_p,
4180 const char *propname,
4184 svn_fs_t *fs = txn->fs;
4186 SVN_ERR(svn_fs__check_fs(fs, TRUE));
4187 SVN_ERR(svn_fs_fs__txn_proplist(&table, txn, pool));
4189 *value_p = svn_hash_gets(table, propname);
4191 return SVN_NO_ERROR;
4195 svn_fs_fs__begin_txn(svn_fs_txn_t **txn_p,
4203 apr_hash_t *props = apr_hash_make(pool);
4205 SVN_ERR(svn_fs__check_fs(fs, TRUE));
4207 SVN_ERR(svn_fs_fs__create_txn(txn_p, fs, rev, pool));
4209 /* Put a datestamp on the newly created txn, so we always know
4210 exactly how old it is. (This will help sysadmins identify
4211 long-abandoned txns that may need to be manually removed.) When
4212 a txn is promoted to a revision, this property will be
4213 automatically overwritten with a revision datestamp. */
4214 date.data = svn_time_to_cstring(apr_time_now(), pool);
4215 date.len = strlen(date.data);
4217 svn_hash_sets(props, SVN_PROP_REVISION_DATE, &date);
4219 /* Set temporary txn props that represent the requested 'flags'
4221 if (flags & SVN_FS_TXN_CHECK_OOD)
4222 svn_hash_sets(props, SVN_FS__PROP_TXN_CHECK_OOD,
4223 svn_string_create("true", pool));
4225 if (flags & SVN_FS_TXN_CHECK_LOCKS)
4226 svn_hash_sets(props, SVN_FS__PROP_TXN_CHECK_LOCKS,
4227 svn_string_create("true", pool));
4229 if (flags & SVN_FS_TXN_CLIENT_DATE)
4230 svn_hash_sets(props, SVN_FS__PROP_TXN_CLIENT_DATE,
4231 svn_string_create("0", pool));
4233 ftd = (*txn_p)->fsap_data;
4234 return svn_error_trace(set_txn_proplist(fs, &ftd->txn_id, props, pool));