1 /* fs.h : interface to Subversion filesystem, private to libsvn_fs
3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
20 * ====================================================================
23 #ifndef SVN_LIBSVN_FS_FS_H
24 #define SVN_LIBSVN_FS_FS_H
26 #include <apr_pools.h>
28 #include <apr_network_io.h>
33 #include "svn_config.h"
34 #include "private/svn_atomic.h"
35 #include "private/svn_cache.h"
36 #include "private/svn_fs_private.h"
37 #include "private/svn_sqlite.h"
38 #include "private/svn_mutex.h"
44 #endif /* __cplusplus */
47 /*** The filesystem structure. ***/
49 /* Following are defines that specify the textual elements of the
50 native filesystem directories and revision files. */
52 /* Names of special files in the fs_fs filesystem. */
53 #define PATH_FORMAT "format" /* Contains format number */
54 #define PATH_UUID "uuid" /* Contains UUID */
55 #define PATH_CURRENT "current" /* Youngest revision */
56 #define PATH_LOCK_FILE "write-lock" /* Revision lock file */
57 #define PATH_PACK_LOCK_FILE "pack-lock" /* Pack lock file */
58 #define PATH_REVS_DIR "revs" /* Directory of revisions */
59 #define PATH_REVPROPS_DIR "revprops" /* Directory of revprops */
60 #define PATH_TXNS_DIR "transactions" /* Directory of transactions in
61 repos w/o log addressing */
62 #define PATH_NODE_ORIGINS_DIR "node-origins" /* Lazy node-origin cache */
63 #define PATH_TXN_PROTOS_DIR "txn-protorevs" /* Directory of proto-revs */
64 #define PATH_TXN_CURRENT "txn-current" /* File with next txn key */
65 #define PATH_TXN_CURRENT_LOCK "txn-current-lock" /* Lock for txn-current */
66 #define PATH_LOCKS_DIR "locks" /* Directory of locks */
67 #define PATH_MIN_UNPACKED_REV "min-unpacked-rev" /* Oldest revision which
68 has not been packed. */
69 #define PATH_REVPROP_GENERATION "revprop-generation"
70 /* Current revprop generation*/
71 #define PATH_MANIFEST "manifest" /* Manifest file name */
72 #define PATH_PACKED "pack" /* Packed revision data file */
73 #define PATH_EXT_PACKED_SHARD ".pack" /* Extension for packed
75 #define PATH_EXT_L2P_INDEX ".l2p" /* extension of the log-
77 #define PATH_EXT_P2L_INDEX ".p2l" /* extension of the phys-
79 /* If you change this, look at tests/svn_test_fs.c(maybe_install_fsfs_conf) */
80 #define PATH_CONFIG "fsfs.conf" /* Configuration */
82 /* Names of special files and file extensions for transactions */
83 #define PATH_CHANGES "changes" /* Records changes made so far */
84 #define PATH_TXN_PROPS "props" /* Transaction properties */
85 #define PATH_TXN_PROPS_FINAL "props-final" /* Final transaction properties
86 before moving to revprops */
87 #define PATH_NEXT_IDS "next-ids" /* Next temporary ID assignments */
88 #define PATH_PREFIX_NODE "node." /* Prefix for node filename */
89 #define PATH_EXT_TXN ".txn" /* Extension of txn dir */
90 #define PATH_EXT_CHILDREN ".children" /* Extension for dir contents */
91 #define PATH_EXT_PROPS ".props" /* Extension for node props */
92 #define PATH_EXT_REV ".rev" /* Extension of protorev file */
93 #define PATH_EXT_REV_LOCK ".rev-lock" /* Extension of protorev lock file */
94 #define PATH_TXN_ITEM_INDEX "itemidx" /* File containing the current item
96 #define PATH_INDEX "index" /* name of index files w/o ext */
98 /* Names of files in legacy FS formats */
99 #define PATH_REV "rev" /* Proto rev file */
100 #define PATH_REV_LOCK "rev-lock" /* Proto rev (write) lock file */
102 /* Names of sections and options in fsfs.conf. */
103 #define CONFIG_SECTION_CACHES "caches"
104 #define CONFIG_OPTION_FAIL_STOP "fail-stop"
105 #define CONFIG_SECTION_REP_SHARING "rep-sharing"
106 #define CONFIG_OPTION_ENABLE_REP_SHARING "enable-rep-sharing"
107 #define CONFIG_SECTION_DELTIFICATION "deltification"
108 #define CONFIG_OPTION_ENABLE_DIR_DELTIFICATION "enable-dir-deltification"
109 #define CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION "enable-props-deltification"
110 #define CONFIG_OPTION_MAX_DELTIFICATION_WALK "max-deltification-walk"
111 #define CONFIG_OPTION_MAX_LINEAR_DELTIFICATION "max-linear-deltification"
112 #define CONFIG_OPTION_COMPRESSION_LEVEL "compression-level"
113 #define CONFIG_SECTION_PACKED_REVPROPS "packed-revprops"
114 #define CONFIG_OPTION_REVPROP_PACK_SIZE "revprop-pack-size"
115 #define CONFIG_OPTION_COMPRESS_PACKED_REVPROPS "compress-packed-revprops"
116 #define CONFIG_SECTION_IO "io"
117 #define CONFIG_OPTION_BLOCK_SIZE "block-size"
118 #define CONFIG_OPTION_L2P_PAGE_SIZE "l2p-page-size"
119 #define CONFIG_OPTION_P2L_PAGE_SIZE "p2l-page-size"
120 #define CONFIG_SECTION_DEBUG "debug"
121 #define CONFIG_OPTION_PACK_AFTER_COMMIT "pack-after-commit"
123 /* The format number of this filesystem.
124 This is independent of the repository format number, and
125 independent of any other FS back ends.
127 Note: If you bump this, please update the switch statement in
128 svn_fs_fs__create() as well.
130 #define SVN_FS_FS__FORMAT_NUMBER 7
132 /* The minimum format number that supports svndiff version 1. */
133 #define SVN_FS_FS__MIN_SVNDIFF1_FORMAT 2
135 /* The minimum format number that supports transaction ID generation
136 using a transaction sequence in the txn-current file. */
137 #define SVN_FS_FS__MIN_TXN_CURRENT_FORMAT 3
139 /* The minimum format number that supports the "layout" filesystem
141 #define SVN_FS_FS__MIN_LAYOUT_FORMAT_OPTION_FORMAT 3
143 /* The minimum format number that stores protorevs in a separate directory. */
144 #define SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT 3
146 /* The minimum format number that doesn't keep node and copy ID counters. */
147 #define SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT 3
149 /* The minimum format number that maintains minfo-here and minfo-count
151 #define SVN_FS_FS__MIN_MERGEINFO_FORMAT 3
153 /* The minimum format number that allows rep sharing. */
154 #define SVN_FS_FS__MIN_REP_SHARING_FORMAT 4
156 /* The minimum format number that supports packed shards. */
157 #define SVN_FS_FS__MIN_PACKED_FORMAT 4
159 /* The minimum format number that stores node kinds in changed-paths lists. */
160 #define SVN_FS_FS__MIN_KIND_IN_CHANGED_FORMAT 4
162 /* 1.8 deltification options should work with any FSFS repo but to avoid
163 * issues with very old servers, restrict those options to the 1.6+ format*/
164 #define SVN_FS_FS__MIN_DELTIFICATION_FORMAT 4
166 /* The 1.7-dev format, never released, that packed revprops into SQLite
168 #define SVN_FS_FS__PACKED_REVPROP_SQLITE_DEV_FORMAT 5
170 /* The minimum format number that supports packed revprops. */
171 #define SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT 6
173 /* The minimum format number that supports packed revprops. */
174 #define SVN_FS_FS__MIN_LOG_ADDRESSING_FORMAT 7
176 /* Minimum format number that providing a separate lock file for pack ops */
177 #define SVN_FS_FS__MIN_PACK_LOCK_FORMAT 7
179 /* Minimum format number that stores mergeinfo-mode flag in changed paths */
180 #define SVN_FS_FS__MIN_MERGEINFO_IN_CHANGED_FORMAT 7
182 /* Minimum format number that supports per-instance filesystem IDs. */
183 #define SVN_FS_FS__MIN_INSTANCE_ID_FORMAT 7
185 /* The minimum format number that supports a configuration file (fsfs.conf) */
186 #define SVN_FS_FS__MIN_CONFIG_FILE 4
188 /* On most operating systems apr implements file locks per process, not
189 per file. On Windows apr implements the locking as per file handle
190 locks, so we don't have to add our own mutex for just in-process
192 #if APR_HAS_THREADS && !defined(WIN32)
193 #define SVN_FS_FS__USE_LOCK_MUTEX 1
195 #define SVN_FS_FS__USE_LOCK_MUTEX 0
198 /* Private FSFS-specific data shared between all svn_txn_t objects that
199 relate to a particular transaction in a filesystem (as identified
200 by transaction id and filesystem UUID). Objects of this type are
201 allocated in their own subpool of the common pool. */
202 typedef struct fs_fs_shared_txn_data_t
204 /* The next transaction in the list, or NULL if there is no following
206 struct fs_fs_shared_txn_data_t *next;
208 /* ID of this transaction. */
209 svn_fs_fs__id_part_t txn_id;
211 /* Whether the transaction's prototype revision file is locked for
212 writing by any thread in this process (including the current
213 thread; recursive locks are not permitted). This is effectively
214 a non-recursive mutex. */
215 svn_boolean_t being_written;
217 /* The pool in which this object has been allocated; a subpool of the
220 } fs_fs_shared_txn_data_t;
222 /* Private FSFS-specific data shared between all svn_fs_t objects that
223 relate to a particular filesystem, as identified by filesystem UUID.
224 Objects of this type are allocated in the common pool. */
225 typedef struct fs_fs_shared_data_t
227 /* A list of shared transaction objects for each transaction that is
228 currently active, or NULL if none are. All access to this list,
229 including the contents of the objects stored in it, is synchronised
230 under TXN_LIST_LOCK. */
231 fs_fs_shared_txn_data_t *txns;
233 /* A free transaction object, or NULL if there is no free object.
234 Access to this object is synchronised under TXN_LIST_LOCK. */
235 fs_fs_shared_txn_data_t *free_txn;
237 /* The following lock must be taken out in reverse order of their
238 declaration here. Any subset may be acquired and held at any given
239 time but their relative acquisition order must not change.
241 (lock 'txn-current' before 'pack' before 'write' before 'txn-list') */
243 /* A lock for intra-process synchronization when accessing the TXNS list. */
244 svn_mutex__t *txn_list_lock;
246 /* A lock for intra-process synchronization when grabbing the
247 repository write lock. */
248 svn_mutex__t *fs_write_lock;
250 /* A lock for intra-process synchronization when grabbing the
251 repository pack operation lock. */
252 svn_mutex__t *fs_pack_lock;
254 /* A lock for intra-process synchronization when locking the
256 svn_mutex__t *txn_current_lock;
258 /* The common pool, under which this object is allocated, subpools
259 of which are used to allocate the transaction objects. */
260 apr_pool_t *common_pool;
261 } fs_fs_shared_data_t;
263 /* Data structure for the 1st level DAG node cache. */
264 typedef struct fs_fs_dag_cache_t fs_fs_dag_cache_t;
266 /* Key type for all caches that use revision + offset / counter as key.
268 Note: Cache keys should be 16 bytes for best performance and there
269 should be no padding. */
270 typedef struct pair_cache_key_t
272 /* The object's revision. Use the 64 data type to prevent padding. */
273 apr_int64_t revision;
275 /* Sub-address: item index, revprop generation, packed flag, etc. */
279 /* Key type that identifies a txdelta window.
281 Note: Cache keys should require no padding. */
282 typedef struct window_cache_key_t
284 /* The object's revision. Use the 64 data type to prevent padding. */
285 apr_int64_t revision;
287 /* Window number within that representation. */
288 apr_int64_t chunk_index;
290 /* Item index of the representation */
291 apr_uint64_t item_index;
292 } window_cache_key_t;
294 /* Private (non-shared) FSFS-specific data for each svn_fs_t object.
295 Any caches in here may be NULL. */
296 typedef struct fs_fs_data_t
298 /* The format number of this FS. */
301 /* The maximum number of files to store per directory (for sharded
302 layouts) or zero (for linear layouts). */
303 int max_files_per_dir;
305 /* If set, this FS is using logical addressing. Otherwise, it is using
306 physical addressing. */
307 svn_boolean_t use_log_addressing;
309 /* Rev / pack file read granularity in bytes. */
310 apr_int64_t block_size;
312 /* Capacity in entries of log-to-phys index pages */
313 apr_int64_t l2p_page_size;
315 /* Rev / pack file granularity (in bytes) covered by a single phys-to-log
317 apr_int64_t p2l_page_size;
319 /* If set, parse and cache *all* data of each block that we read
320 * (not just the one bit that we need, atm). */
321 svn_boolean_t use_block_read;
323 /* The revision that was youngest, last time we checked. */
324 svn_revnum_t youngest_rev_cache;
326 /* Caches of immutable data. (Note that these may be shared between
327 multiple svn_fs_t's for the same filesystem.) */
329 /* Access to the configured memcached instances. May be NULL. */
330 svn_memcache_t *memcache;
332 /* If TRUE, don't ignore any cache-related errors. If FALSE, errors from
333 e.g. memcached may be ignored as caching is an optional feature. */
334 svn_boolean_t fail_stop;
336 /* A cache of revision root IDs, mapping from (svn_revnum_t *) to
337 (svn_fs_id_t *). (Not threadsafe.) */
338 svn_cache__t *rev_root_id_cache;
340 /* Caches native dag_node_t* instances and acts as a 1st level cache */
341 fs_fs_dag_cache_t *dag_node_cache;
343 /* DAG node cache for immutable nodes. Maps (revision, fspath)
344 to (dag_node_t *). This is the 2nd level cache for DAG nodes. */
345 svn_cache__t *rev_node_cache;
347 /* A cache of the contents of immutable directories; maps from
348 unparsed FS ID to a apr_hash_t * mapping (const char *) dirent
349 names to (svn_fs_dirent_t *). */
350 svn_cache__t *dir_cache;
352 /* Fulltext cache; currently only used with memcached. Maps from
353 rep key (revision/offset) to svn_stringbuf_t. */
354 svn_cache__t *fulltext_cache;
356 /* Node properties cache. Maps from rep key to apr_hash_t. */
357 svn_cache__t *properties_cache;
359 /* Pack manifest cache; a cache mapping (svn_revnum_t) shard number to
360 a manifest; and a manifest is a mapping from (svn_revnum_t) revision
361 number offset within a shard to (apr_off_t) byte-offset in the
362 respective pack file. */
363 svn_cache__t *packed_offset_cache;
365 /* Cache for svn_fs_fs__raw_cached_window_t objects; the key is
366 window_cache_key_t. */
367 svn_cache__t *raw_window_cache;
369 /* Cache for txdelta_window_t objects; the key is window_cache_key_t */
370 svn_cache__t *txdelta_window_cache;
372 /* Cache for combined windows as svn_stringbuf_t objects;
373 the key is window_cache_key_t */
374 svn_cache__t *combined_window_cache;
376 /* Cache for node_revision_t objects; the key is (revision, item_index) */
377 svn_cache__t *node_revision_cache;
379 /* Cache for change lists as APR arrays of change_t * objects; the key
381 svn_cache__t *changes_cache;
383 /* Cache for svn_fs_fs__rep_header_t objects; the key is a
384 (revision, item index) pair */
385 svn_cache__t *rep_header_cache;
387 /* Cache for svn_mergeinfo_t objects; the key is a combination of
388 revision, inheritance flags and path. */
389 svn_cache__t *mergeinfo_cache;
391 /* Cache for presence of svn_mergeinfo_t on a noderev; the key is a
392 combination of revision, inheritance flags and path; value is "1"
393 if the node has mergeinfo, "0" if it doesn't. */
394 svn_cache__t *mergeinfo_existence_cache;
396 /* Cache for l2p_header_t objects; the key is (revision, is-packed).
397 Will be NULL for pre-format7 repos */
398 svn_cache__t *l2p_header_cache;
400 /* Cache for l2p_page_t objects; the key is svn_fs_fs__page_cache_key_t.
401 Will be NULL for pre-format7 repos */
402 svn_cache__t *l2p_page_cache;
404 /* Cache for p2l_header_t objects; the key is (revision, is-packed).
405 Will be NULL for pre-format7 repos */
406 svn_cache__t *p2l_header_cache;
408 /* Cache for apr_array_header_t objects containing svn_fs_fs__p2l_entry_t
409 elements; the key is svn_fs_fs__page_cache_key_t.
410 Will be NULL for pre-format7 repos */
411 svn_cache__t *p2l_page_cache;
413 /* TRUE while the we hold a lock on the write lock file. */
414 svn_boolean_t has_write_lock;
416 /* If set, there are or have been more than one concurrent transaction */
417 svn_boolean_t concurrent_transactions;
419 /* Temporary cache for changed directories yet to be committed; maps from
420 unparsed FS ID to ###x. NULL outside transactions. */
421 svn_cache__t *txn_dir_cache;
423 /* Data shared between all svn_fs_t objects for a given filesystem. */
424 fs_fs_shared_data_t *shared;
426 /* The sqlite database used for rep caching. */
427 svn_sqlite__db_t *rep_cache_db;
429 /* Thread-safe boolean */
430 svn_atomic_t rep_cache_db_opened;
432 /* The oldest revision not in a pack file. It also applies to revprops
433 * if revprop packing has been enabled by the FSFS format version. */
434 svn_revnum_t min_unpacked_rev;
436 /* Whether rep-sharing is supported by the filesystem
437 * and allowed by the configuration. */
438 svn_boolean_t rep_sharing_allowed;
440 /* File size limit in bytes up to which multiple revprops shall be packed
441 * into a single file. */
442 apr_int64_t revprop_pack_size;
444 /* Whether packed revprop files shall be compressed. */
445 svn_boolean_t compress_packed_revprops;
447 /* Whether directory nodes shall be deltified just like file nodes. */
448 svn_boolean_t deltify_directories;
450 /* Whether nodes properties shall be deltified. */
451 svn_boolean_t deltify_properties;
453 /* Restart deltification histories after each multiple of this value */
454 apr_int64_t max_deltification_walk;
456 /* Maximum number of length of the linear part at the top of the
457 * deltification history after which skip deltas will be used. */
458 apr_int64_t max_linear_deltification;
460 /* Compression level to use with txdelta storage format in new revs. */
461 int delta_compression_level;
463 /* Pack after every commit. */
464 svn_boolean_t pack_after_commit;
466 /* Per-instance filesystem ID, which provides an additional level of
467 uniqueness for filesystems that share the same UUID, but should
468 still be distinguishable (e.g. backups produced by svn_fs_hotcopy()
469 or dump / load cycles). */
470 const char *instance_id;
472 /* Pointer to svn_fs_open. */
473 svn_error_t *(*svn_fs_open_)(svn_fs_t **, const char *, apr_hash_t *,
474 apr_pool_t *, apr_pool_t *);
478 /*** Filesystem Transaction ***/
479 typedef struct transaction_t
481 /* node revision id of the root node. */
482 const svn_fs_id_t *root_id;
484 /* node revision id of the node which is the root of the revision
485 upon which this txn is base. (unfinished only) */
486 const svn_fs_id_t *base_id;
488 /* copies list (const char * copy_ids), or NULL if there have been
489 no copies in this transaction. */
490 apr_array_header_t *copies;
495 /*** Representation ***/
496 /* If you add fields to this, check to see if you need to change
497 * svn_fs_fs__rep_copy. */
498 typedef struct representation_t
500 /* Checksums digests for the contents produced by this representation.
501 This checksum is for the contents the rep shows to consumers,
502 regardless of how the rep stores the data under the hood. It is
503 independent of the storage (fulltext, delta, whatever).
505 If has_sha1 is FALSE, then for compatibility behave as though this
506 checksum matches the expected checksum.
508 The md5 checksum is always filled, unless this is rep which was
509 retrieved from the rep-cache. The sha1 checksum is only computed on
510 a write, for use with rep-sharing. */
511 svn_boolean_t has_sha1;
512 unsigned char sha1_digest[APR_SHA1_DIGESTSIZE];
513 unsigned char md5_digest[APR_MD5_DIGESTSIZE];
515 /* Revision where this representation is located. */
516 svn_revnum_t revision;
518 /* Item index with the the revision. */
519 apr_uint64_t item_index;
521 /* The size of the representation in bytes as seen in the revision
525 /* The size of the fulltext of the representation. If this is 0,
526 * the fulltext size is equal to representation size in the rev file, */
527 svn_filesize_t expanded_size;
529 /* Is this a representation (still) within a transaction? */
530 svn_fs_fs__id_part_t txn_id;
532 /* For rep-sharing, we need a way of uniquifying node-revs which share the
533 same representation (see svn_fs_fs__noderev_same_rep_key() ). So, we
534 store the original txn of the node rev (not the rep!), along with some
535 intra-node uniqification content. */
538 /* unique context, i.e. txn ID, in which the noderev (!) got created */
539 svn_fs_fs__id_part_t noderev_txn_id;
541 /* unique value within that txn */
547 /*** Node-Revision ***/
548 /* If you add fields to this, check to see if you need to change
549 * copy_node_revision in dag.c. */
550 typedef struct node_revision_t
553 svn_node_kind_t kind;
555 /* The node-id for this node-rev. */
556 const svn_fs_id_t *id;
558 /* predecessor node revision id, or NULL if there is no predecessor
559 for this node revision */
560 const svn_fs_id_t *predecessor_id;
562 /* If this node-rev is a copy, where was it copied from? */
563 const char *copyfrom_path;
564 svn_revnum_t copyfrom_rev;
566 /* Helper for history tracing, root of the parent tree from whence
567 this node-rev was copied. */
568 svn_revnum_t copyroot_rev;
569 const char *copyroot_path;
571 /* number of predecessors this node revision has (recursively), or
572 -1 if not known (for backward compatibility). */
573 int predecessor_count;
575 /* representation key for this node's properties. may be NULL if
576 there are no properties. */
577 representation_t *prop_rep;
579 /* representation for this node's data. may be NULL if there is
581 representation_t *data_rep;
583 /* path at which this node first came into existence. */
584 const char *created_path;
586 /* is this the unmodified root of a transaction? */
587 svn_boolean_t is_fresh_txn_root;
589 /* Number of nodes with svn:mergeinfo properties that are
590 descendants of this node (including it itself) */
591 apr_int64_t mergeinfo_count;
593 /* Does this node itself have svn:mergeinfo? */
594 svn_boolean_t has_mergeinfo;
600 typedef struct change_t
602 /* Path of the change. */
605 /* API compatible change description */
606 svn_fs_path_change2_t info;
612 #endif /* __cplusplus */
614 #endif /* SVN_LIBSVN_FS_FS_H */