contrib/subversion/subversion/libsvn_fs_fs/revprops.c

   1 /* revprops.c --- everything needed to handle revprops in FSFS
   2  *
   3  * ====================================================================
   4  *    Licensed to the Apache Software Foundation (ASF) under one
   5  *    or more contributor license agreements.  See the NOTICE file
   6  *    distributed with this work for additional information
   7  *    regarding copyright ownership.  The ASF licenses this file
   8  *    to you under the Apache License, Version 2.0 (the
   9  *    "License"); you may not use this file except in compliance
  10  *    with the License.  You may obtain a copy of the License at
  11  *
  12  *      http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  *    Unless required by applicable law or agreed to in writing,
  15  *    software distributed under the License is distributed on an
  16  *    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  *    KIND, either express or implied.  See the License for the
  18  *    specific language governing permissions and limitations
  19  *    under the License.
  20  * ====================================================================
  21  */
  22
  23 #include <assert.h>
  24
  25 #include "svn_pools.h"
  26 #include "svn_hash.h"
  27 #include "svn_dirent_uri.h"
  28
  29 #include "fs_fs.h"
  30 #include "revprops.h"
  31 #include "util.h"
  32
  33 #include "private/svn_subr_private.h"
  34 #include "private/svn_string_private.h"
  35 #include "../libsvn_fs/fs-loader.h"
  36
  37 #include "svn_private_config.h"
  38
  39 /* Give writing processes 10 seconds to replace an existing revprop
  40    file with a new one. After that time, we assume that the writing
  41    process got aborted and that we have re-read revprops. */
  42 #define REVPROP_CHANGE_TIMEOUT (10 * 1000000)
  43
  44 svn_error_t *
  45 svn_fs_fs__upgrade_pack_revprops(svn_fs_t *fs,
  46                                  svn_fs_upgrade_notify_t notify_func,
  47                                  void *notify_baton,
  48                                  svn_cancel_func_t cancel_func,
  49                                  void *cancel_baton,
  50                                  apr_pool_t *scratch_pool)
  51 {
  52   fs_fs_data_t *ffd = fs->fsap_data;
  53   const char *revprops_shard_path;
  54   const char *revprops_pack_file_dir;
  55   apr_int64_t shard;
  56   apr_int64_t first_unpacked_shard
  57     =  ffd->min_unpacked_rev / ffd->max_files_per_dir;
  58
  59   apr_pool_t *iterpool = svn_pool_create(scratch_pool);
  60   const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR,
  61                                               scratch_pool);
  62   int compression_level = ffd->compress_packed_revprops
  63                            ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
  64                            : SVN_DELTA_COMPRESSION_LEVEL_NONE;
  65
  66   /* first, pack all revprops shards to match the packed revision shards */
  67   for (shard = 0; shard < first_unpacked_shard; ++shard)
  68     {
  69       svn_pool_clear(iterpool);
  70
  71       revprops_pack_file_dir = svn_dirent_join(revsprops_dir,
  72                    apr_psprintf(iterpool,
  73                                 "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
  74                                 shard),
  75                    iterpool);
  76       revprops_shard_path = svn_dirent_join(revsprops_dir,
  77                        apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
  78                        iterpool);
  79
  80       SVN_ERR(svn_fs_fs__pack_revprops_shard(revprops_pack_file_dir,
  81                                              revprops_shard_path,
  82                                              shard, ffd->max_files_per_dir,
  83                                              (int)(0.9 * ffd->revprop_pack_size),
  84                                              compression_level,
  85                                              cancel_func, cancel_baton,
  86                                              iterpool));
  87       if (notify_func)
  88         SVN_ERR(notify_func(notify_baton, shard,
  89                             svn_fs_upgrade_pack_revprops, iterpool));
  90     }
  91
  92   svn_pool_destroy(iterpool);
  93
  94   return SVN_NO_ERROR;
  95 }
  96
  97 svn_error_t *
  98 svn_fs_fs__upgrade_cleanup_pack_revprops(svn_fs_t *fs,
  99                                          svn_fs_upgrade_notify_t notify_func,
 100                                          void *notify_baton,
 101                                          svn_cancel_func_t cancel_func,
 102                                          void *cancel_baton,
 103                                          apr_pool_t *scratch_pool)
 104 {
 105   fs_fs_data_t *ffd = fs->fsap_data;
 106   const char *revprops_shard_path;
 107   apr_int64_t shard;
 108   apr_int64_t first_unpacked_shard
 109     =  ffd->min_unpacked_rev / ffd->max_files_per_dir;
 110
 111   apr_pool_t *iterpool = svn_pool_create(scratch_pool);
 112   const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR,
 113                                               scratch_pool);
 114
 115   /* delete the non-packed revprops shards afterwards */
 116   for (shard = 0; shard < first_unpacked_shard; ++shard)
 117     {
 118       svn_pool_clear(iterpool);
 119
 120       revprops_shard_path = svn_dirent_join(revsprops_dir,
 121                        apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
 122                        iterpool);
 123       SVN_ERR(svn_fs_fs__delete_revprops_shard(revprops_shard_path,
 124                                                shard,
 125                                                ffd->max_files_per_dir,
 126                                                cancel_func, cancel_baton,
 127                                                iterpool));
 128       if (notify_func)
 129         SVN_ERR(notify_func(notify_baton, shard,
 130                             svn_fs_upgrade_cleanup_revprops, iterpool));
 131     }
 132
 133   svn_pool_destroy(iterpool);
 134
 135   return SVN_NO_ERROR;
 136 }
 137
 138 /* Container for all data required to access the packed revprop file
 139  * for a given REVISION.  This structure will be filled incrementally
 140  * by read_pack_revprops() its sub-routines.
 141  */
 142 typedef struct packed_revprops_t
 143 {
 144   /* revision number to read (not necessarily the first in the pack) */
 145   svn_revnum_t revision;
 146
 147   /* current revprop generation. Used when populating the revprop cache */
 148   apr_int64_t generation;
 149
 150   /* the actual revision properties */
 151   apr_hash_t *properties;
 152
 153   /* their size when serialized to a single string
 154    * (as found in PACKED_REVPROPS) */
 155   apr_size_t serialized_size;
 156
 157
 158   /* name of the pack file (without folder path) */
 159   const char *filename;
 160
 161   /* packed shard folder path */
 162   const char *folder;
 163
 164   /* sum of values in SIZES */
 165   apr_size_t total_size;
 166
 167   /* first revision in the pack (>= MANIFEST_START) */
 168   svn_revnum_t start_revision;
 169
 170   /* size of the revprops in PACKED_REVPROPS */
 171   apr_array_header_t *sizes;
 172
 173   /* offset of the revprops in PACKED_REVPROPS */
 174   apr_array_header_t *offsets;
 175
 176
 177   /* concatenation of the serialized representation of all revprops
 178    * in the pack, i.e. the pack content without header and compression */
 179   svn_stringbuf_t *packed_revprops;
 180
 181   /* First revision covered by MANIFEST.
 182    * Will equal the shard start revision or 1, for the 1st shard. */
 183   svn_revnum_t manifest_start;
 184
 185   /* content of the manifest.
 186    * Maps long(rev - MANIFEST_START) to const char* pack file name */
 187   apr_array_header_t *manifest;
 188 } packed_revprops_t;
 189
 190 /* Parse the serialized revprops in CONTENT and return them in *PROPERTIES.
 191  * Also, put them into the revprop cache, if activated, for future use.
 192  * Three more parameters are being used to update the revprop cache: FS is
 193  * our file system, the revprops belong to REVISION and the global revprop
 194  * GENERATION is used as well.
 195  *
 196  * The returned hash will be allocated in POOL, SCRATCH_POOL is being used
 197  * for temporary allocations.
 198  */
 199 static svn_error_t *
 200 parse_revprop(apr_hash_t **properties,
 201               svn_fs_t *fs,
 202               svn_revnum_t revision,
 203               apr_int64_t generation,
 204               svn_string_t *content,
 205               apr_pool_t *pool,
 206               apr_pool_t *scratch_pool)
 207 {
 208   svn_stream_t *stream = svn_stream_from_string(content, scratch_pool);
 209   *properties = apr_hash_make(pool);
 210
 211   SVN_ERR_W(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool),
 212             apr_psprintf(scratch_pool, "Failed to parse revprops for r%ld.",
 213                          revision));
 214
 215   return SVN_NO_ERROR;
 216 }
 217
 218 /* Read the non-packed revprops for revision REV in FS, put them into the
 219  * revprop cache if activated and return them in *PROPERTIES.  GENERATION
 220  * is the current revprop generation.
 221  *
 222  * If the data could not be read due to an otherwise recoverable error,
 223  * leave *PROPERTIES unchanged. No error will be returned in that case.
 224  *
 225  * Allocations will be done in POOL.
 226  */
 227 static svn_error_t *
 228 read_non_packed_revprop(apr_hash_t **properties,
 229                         svn_fs_t *fs,
 230                         svn_revnum_t rev,
 231                         apr_int64_t generation,
 232                         apr_pool_t *pool)
 233 {
 234   svn_stringbuf_t *content = NULL;
 235   apr_pool_t *iterpool = svn_pool_create(pool);
 236   svn_boolean_t missing = FALSE;
 237   int i;
 238
 239   for (i = 0;
 240        i < SVN_FS_FS__RECOVERABLE_RETRY_COUNT && !missing && !content;
 241        ++i)
 242     {
 243       svn_pool_clear(iterpool);
 244       SVN_ERR(svn_fs_fs__try_stringbuf_from_file(&content,
 245                               &missing,
 246                               svn_fs_fs__path_revprops(fs, rev, iterpool),
 247                               i + 1 < SVN_FS_FS__RECOVERABLE_RETRY_COUNT ,
 248                               iterpool));
 249     }
 250
 251   if (content)
 252     SVN_ERR(parse_revprop(properties, fs, rev, generation,
 253                           svn_stringbuf__morph_into_string(content),
 254                           pool, iterpool));
 255
 256   svn_pool_clear(iterpool);
 257
 258   return SVN_NO_ERROR;
 259 }
 260
 261 /* Return the minimum length of any packed revprop file name in REVPROPS. */
 262 static apr_size_t
 263 get_min_filename_len(packed_revprops_t *revprops)
 264 {
 265   char number_buffer[SVN_INT64_BUFFER_SIZE];
 266
 267   /* The revprop filenames have the format <REV>.<COUNT> - with <REV> being
 268    * at least the first rev in the shard and <COUNT> having at least one
 269    * digit.  Thus, the minimum is 2 + #decimal places in the start rev.
 270    */
 271   return svn__i64toa(number_buffer, revprops->manifest_start) + 2;
 272 }
 273
 274 /* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST
 275  * members. Use POOL for allocating results and SCRATCH_POOL for temporaries.
 276  */
 277 static svn_error_t *
 278 get_revprop_packname(svn_fs_t *fs,
 279                      packed_revprops_t *revprops,
 280                      apr_pool_t *pool,
 281                      apr_pool_t *scratch_pool)
 282 {
 283   fs_fs_data_t *ffd = fs->fsap_data;
 284   svn_stringbuf_t *content = NULL;
 285   const char *manifest_file_path;
 286   int idx, rev_count;
 287   char *buffer, *buffer_end;
 288   const char **filenames, **filenames_end;
 289   apr_size_t min_filename_len;
 290
 291   /* Determine the dimensions. Rev 0 is excluded from the first shard. */
 292   rev_count = ffd->max_files_per_dir;
 293   revprops->manifest_start
 294     = revprops->revision - (revprops->revision % rev_count);
 295   if (revprops->manifest_start == 0)
 296     {
 297       ++revprops->manifest_start;
 298       --rev_count;
 299     }
 300
 301   revprops->manifest = apr_array_make(pool, rev_count, sizeof(const char*));
 302
 303   /* No line in the file can be less than this number of chars long. */
 304   min_filename_len = get_min_filename_len(revprops);
 305
 306   /* Read the content of the manifest file */
 307   revprops->folder
 308     = svn_fs_fs__path_revprops_pack_shard(fs, revprops->revision, pool);
 309   manifest_file_path
 310     = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool);
 311
 312   SVN_ERR(svn_fs_fs__read_content(&content, manifest_file_path, pool));
 313
 314   /* There CONTENT must have a certain minimal size and there no
 315    * unterminated lines at the end of the file.  Both guarantees also
 316    * simplify the parser loop below.
 317    */
 318   if (   content->len < rev_count * (min_filename_len + 1)
 319       || content->data[content->len - 1] != '\n')
 320     return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
 321                              _("Packed revprop manifest for r%ld not "
 322                                "properly terminated"), revprops->revision);
 323
 324   /* Chop (parse) the manifest CONTENT into filenames, one per line.
 325    * We only have to replace all newlines with NUL and add all line
 326    * starts to REVPROPS->MANIFEST.
 327    *
 328    * There must be exactly REV_COUNT lines and that is the number of
 329    * lines we parse from BUFFER to FILENAMES.  Set the end pointer for
 330    * the source BUFFER such that BUFFER+MIN_FILENAME_LEN is still valid
 331    * BUFFER_END is always valid due to CONTENT->LEN > MIN_FILENAME_LEN.
 332    *
 333    * Please note that this loop is performance critical for e.g. 'svn log'.
 334    * It is run 1000x per revprop access, i.e. per revision and about
 335    * 50 million times per sec (and CPU core).
 336    */
 337   for (filenames = (const char **)revprops->manifest->elts,
 338        filenames_end = filenames + rev_count,
 339        buffer = content->data,
 340        buffer_end = buffer + content->len - min_filename_len;
 341        (filenames < filenames_end) && (buffer < buffer_end);
 342        ++filenames)
 343     {
 344       /* BUFFER always points to the start of the next line / filename. */
 345       *filenames = buffer;
 346
 347       /* Find the next EOL.  This is guaranteed to stay within the CONTENT
 348        * buffer because we left enough room after BUFFER_END and we know
 349        * we will always see a newline as the last non-NUL char. */
 350       buffer += min_filename_len;
 351       while (*buffer != '\n')
 352         ++buffer;
 353
 354       /* Found EOL.  Turn it into the filename terminator and move BUFFER
 355        * to the start of the next line or CONTENT buffer end. */
 356       *buffer = '\0';
 357       ++buffer;
 358     }
 359
 360   /* We must have reached the end of both buffers. */
 361   if (buffer < content->data + content->len)
 362     return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
 363                              _("Packed revprop manifest for r%ld "
 364                                "has too many entries"), revprops->revision);
 365
 366   if (filenames < filenames_end)
 367     return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
 368                              _("Packed revprop manifest for r%ld "
 369                                "has too few entries"), revprops->revision);
 370
 371   /* The target array has now exactly one entry per revision. */
 372   revprops->manifest->nelts = rev_count;
 373
 374   /* Now get the file name */
 375   idx = (int)(revprops->revision - revprops->manifest_start);
 376   revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*);
 377
 378   return SVN_NO_ERROR;
 379 }
 380
 381 /* Return TRUE, if revision R1 and R2 refer to the same shard in FS.
 382  */
 383 static svn_boolean_t
 384 same_shard(svn_fs_t *fs,
 385            svn_revnum_t r1,
 386            svn_revnum_t r2)
 387 {
 388   fs_fs_data_t *ffd = fs->fsap_data;
 389   return (r1 / ffd->max_files_per_dir) == (r2 / ffd->max_files_per_dir);
 390 }
 391
 392 /* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS,
 393  * fill the START_REVISION member, and make PACKED_REVPROPS point to the
 394  * first serialized revprop.  If READ_ALL is set, initialize the SIZES
 395  * and OFFSETS members as well.
 396  *
 397  * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as
 398  * well as the SERIALIZED_SIZE member.  If revprop caching has been
 399  * enabled, parse all revprops in the pack and cache them.
 400  */
 401 static svn_error_t *
 402 parse_packed_revprops(svn_fs_t *fs,
 403                       packed_revprops_t *revprops,
 404                       svn_boolean_t read_all,
 405                       apr_pool_t *pool,
 406                       apr_pool_t *scratch_pool)
 407 {
 408   svn_stream_t *stream;
 409   apr_int64_t first_rev, count, i;
 410   apr_off_t offset;
 411   const char *header_end;
 412   apr_pool_t *iterpool = svn_pool_create(scratch_pool);
 413
 414   /* decompress (even if the data is only "stored", there is still a
 415    * length header to remove) */
 416   svn_stringbuf_t *compressed = revprops->packed_revprops;
 417   svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(pool);
 418   SVN_ERR(svn__decompress(compressed, uncompressed, APR_SIZE_MAX));
 419
 420   /* read first revision number and number of revisions in the pack */
 421   stream = svn_stream_from_stringbuf(uncompressed, scratch_pool);
 422   SVN_ERR(svn_fs_fs__read_number_from_stream(&first_rev, NULL, stream,
 423                                              iterpool));
 424   SVN_ERR(svn_fs_fs__read_number_from_stream(&count, NULL, stream,
 425                                              iterpool));
 426
 427   /* Check revision range for validity. */
 428   if (   !same_shard(fs, revprops->revision, first_rev)
 429       || !same_shard(fs, revprops->revision, first_rev + count - 1)
 430       || count < 1)
 431     return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
 432                              _("Revprop pack for revision r%ld"
 433                                " contains revprops for r%ld .. r%ld"),
 434                              revprops->revision,
 435                              (svn_revnum_t)first_rev,
 436                              (svn_revnum_t)(first_rev + count -1));
 437
 438   /* Since start & end are in the same shard, it is enough to just test
 439    * the FIRST_REV for being actually packed.  That will also cover the
 440    * special case of rev 0 never being packed. */
 441   if (!svn_fs_fs__is_packed_revprop(fs, first_rev))
 442     return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
 443                              _("Revprop pack for revision r%ld"
 444                                " starts at non-packed revisions r%ld"),
 445                              revprops->revision, (svn_revnum_t)first_rev);
 446
 447   /* make PACKED_REVPROPS point to the first char after the header.
 448    * This is where the serialized revprops are. */
 449   header_end = strstr(uncompressed->data, "\n\n");
 450   if (header_end == NULL)
 451     return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
 452                             _("Header end not found"));
 453
 454   offset = header_end - uncompressed->data + 2;
 455
 456   revprops->packed_revprops = svn_stringbuf_create_empty(pool);
 457   revprops->packed_revprops->data = uncompressed->data + offset;
 458   revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset);
 459   revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset);
 460
 461   /* STREAM still points to the first entry in the sizes list. */
 462   revprops->start_revision = (svn_revnum_t)first_rev;
 463   if (read_all)
 464     {
 465       /* Init / construct REVPROPS members. */
 466       revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset));
 467       revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset));
 468     }
 469
 470   /* Now parse, revision by revision, the size and content of each
 471    * revisions' revprops. */
 472   for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i)
 473     {
 474       apr_int64_t size;
 475       svn_string_t serialized;
 476       svn_revnum_t revision = (svn_revnum_t)(first_rev + i);
 477       svn_pool_clear(iterpool);
 478
 479       /* read & check the serialized size */
 480       SVN_ERR(svn_fs_fs__read_number_from_stream(&size, NULL, stream,
 481                                                  iterpool));
 482       if (size + offset > (apr_int64_t)revprops->packed_revprops->len)
 483         return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
 484                         _("Packed revprop size exceeds pack file size"));
 485
 486       /* Parse this revprops list, if necessary */
 487       serialized.data = revprops->packed_revprops->data + offset;
 488       serialized.len = (apr_size_t)size;
 489
 490       if (revision == revprops->revision)
 491         {
 492           SVN_ERR(parse_revprop(&revprops->properties, fs, revision,
 493                                 revprops->generation, &serialized,
 494                                 pool, iterpool));
 495           revprops->serialized_size = serialized.len;
 496
 497           /* If we only wanted the revprops for REVISION then we are done. */
 498           if (!read_all)
 499             break;
 500         }
 501
 502       if (read_all)
 503         {
 504           /* fill REVPROPS data structures */
 505           APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len;
 506           APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset;
 507         }
 508       revprops->total_size += serialized.len;
 509
 510       offset += serialized.len;
 511     }
 512
 513   return SVN_NO_ERROR;
 514 }
 515
 516 /* In filesystem FS, read the packed revprops for revision REV into
 517  * *REVPROPS.  Use GENERATION to populate the revprop cache, if enabled.
 518  * If you want to modify revprop contents / update REVPROPS, READ_ALL
 519  * must be set.  Otherwise, only the properties of REV are being provided.
 520  * Allocate data in POOL.
 521  */
 522 static svn_error_t *
 523 read_pack_revprop(packed_revprops_t **revprops,
 524                   svn_fs_t *fs,
 525                   svn_revnum_t rev,
 526                   apr_int64_t generation,
 527                   svn_boolean_t read_all,
 528                   apr_pool_t *pool)
 529 {
 530   apr_pool_t *iterpool = svn_pool_create(pool);
 531   svn_boolean_t missing = FALSE;
 532   svn_error_t *err;
 533   packed_revprops_t *result;
 534   int i;
 535
 536   /* someone insisted that REV is packed. Double-check if necessary */
 537   if (!svn_fs_fs__is_packed_revprop(fs, rev))
 538      SVN_ERR(svn_fs_fs__update_min_unpacked_rev(fs, iterpool));
 539
 540   if (!svn_fs_fs__is_packed_revprop(fs, rev))
 541     return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
 542                               _("No such packed revision %ld"), rev);
 543
 544   /* initialize the result data structure */
 545   result = apr_pcalloc(pool, sizeof(*result));
 546   result->revision = rev;
 547   result->generation = generation;
 548
 549   /* try to read the packed revprops. This may require retries if we have
 550    * concurrent writers. */
 551   for (i = 0;
 552        i < SVN_FS_FS__RECOVERABLE_RETRY_COUNT && !result->packed_revprops;
 553        ++i)
 554     {
 555       const char *file_path;
 556       svn_pool_clear(iterpool);
 557
 558       /* there might have been concurrent writes.
 559        * Re-read the manifest and the pack file.
 560        */
 561       SVN_ERR(get_revprop_packname(fs, result, pool, iterpool));
 562       file_path  = svn_dirent_join(result->folder,
 563                                    result->filename,
 564                                    iterpool);
 565       SVN_ERR(svn_fs_fs__try_stringbuf_from_file(&result->packed_revprops,
 566                                 &missing,
 567                                 file_path,
 568                                 i + 1 < SVN_FS_FS__RECOVERABLE_RETRY_COUNT,
 569                                 pool));
 570     }
 571
 572   /* the file content should be available now */
 573   if (!result->packed_revprops)
 574     return svn_error_createf(SVN_ERR_FS_PACKED_REVPROP_READ_FAILURE, NULL,
 575                   _("Failed to read revprop pack file for r%ld"), rev);
 576
 577   /* parse it. RESULT will be complete afterwards. */
 578   err = parse_packed_revprops(fs, result, read_all, pool, iterpool);
 579   svn_pool_destroy(iterpool);
 580   if (err)
 581     return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
 582                   _("Revprop pack file for r%ld is corrupt"), rev);
 583
 584   *revprops = result;
 585
 586   return SVN_NO_ERROR;
 587 }
 588
 589 /* Read the revprops for revision REV in FS and return them in *PROPERTIES_P.
 590  *
 591  * Allocations will be done in POOL.
 592  */
 593 svn_error_t *
 594 svn_fs_fs__get_revision_proplist(apr_hash_t **proplist_p,
 595                                  svn_fs_t *fs,
 596                                  svn_revnum_t rev,
 597                                  apr_pool_t *pool)
 598 {
 599   fs_fs_data_t *ffd = fs->fsap_data;
 600   apr_int64_t generation = 0;
 601
 602   /* not found, yet */
 603   *proplist_p = NULL;
 604
 605   /* should they be available at all? */
 606   SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, pool));
 607
 608   /* if REV had not been packed when we began, try reading it from the
 609    * non-packed shard.  If that fails, we will fall through to packed
 610    * shard reads. */
 611   if (!svn_fs_fs__is_packed_revprop(fs, rev))
 612     {
 613       svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev,
 614                                                  generation, pool);
 615       if (err)
 616         {
 617           if (!APR_STATUS_IS_ENOENT(err->apr_err)
 618               || ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
 619             return svn_error_trace(err);
 620
 621           svn_error_clear(err);
 622           *proplist_p = NULL; /* in case read_non_packed_revprop changed it */
 623         }
 624     }
 625
 626   /* if revprop packing is available and we have not read the revprops, yet,
 627    * try reading them from a packed shard.  If that fails, REV is most
 628    * likely invalid (or its revprops highly contested). */
 629   if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p)
 630     {
 631       packed_revprops_t *revprops;
 632       SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, FALSE, pool));
 633       *proplist_p = revprops->properties;
 634     }
 635
 636   /* The revprops should have been there. Did we get them? */
 637   if (!*proplist_p)
 638     return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
 639                              _("Could not read revprops for revision %ld"),
 640                              rev);
 641
 642   return SVN_NO_ERROR;
 643 }
 644
 645 /* Serialize the revision property list PROPLIST of revision REV in
 646  * filesystem FS to a non-packed file.  Return the name of that temporary
 647  * file in *TMP_PATH and the file path that it must be moved to in
 648  * *FINAL_PATH.
 649  *
 650  * Use POOL for allocations.
 651  */
 652 static svn_error_t *
 653 write_non_packed_revprop(const char **final_path,
 654                          const char **tmp_path,
 655                          svn_fs_t *fs,
 656                          svn_revnum_t rev,
 657                          apr_hash_t *proplist,
 658                          apr_pool_t *pool)
 659 {
 660   apr_file_t *file;
 661   svn_stream_t *stream;
 662   *final_path = svn_fs_fs__path_revprops(fs, rev, pool);
 663
 664   /* ### do we have a directory sitting around already? we really shouldn't
 665      ### have to get the dirname here. */
 666   SVN_ERR(svn_io_open_unique_file3(&file, tmp_path,
 667                                    svn_dirent_dirname(*final_path, pool),
 668                                    svn_io_file_del_none, pool, pool));
 669   stream = svn_stream_from_aprfile2(file, TRUE, pool);
 670   SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
 671   SVN_ERR(svn_stream_close(stream));
 672
 673   /* Flush temporary file to disk and close it. */
 674   SVN_ERR(svn_io_file_flush_to_disk(file, pool));
 675   SVN_ERR(svn_io_file_close(file, pool));
 676
 677   return SVN_NO_ERROR;
 678 }
 679
 680 /* After writing the new revprop file(s), call this function to move the
 681  * file at TMP_PATH to FINAL_PATH and give it the permissions from
 682  * PERMS_REFERENCE.
 683  *
 684  * Finally, delete all the temporary files given in FILES_TO_DELETE.
 685  * The latter may be NULL.
 686  *
 687  * Use POOL for temporary allocations.
 688  */
 689 static svn_error_t *
 690 switch_to_new_revprop(svn_fs_t *fs,
 691                       const char *final_path,
 692                       const char *tmp_path,
 693                       const char *perms_reference,
 694                       apr_array_header_t *files_to_delete,
 695                       apr_pool_t *pool)
 696 {
 697   SVN_ERR(svn_fs_fs__move_into_place(tmp_path, final_path, perms_reference,
 698                                      pool));
 699
 700   /* Clean up temporary files, if necessary. */
 701   if (files_to_delete)
 702     {
 703       apr_pool_t *iterpool = svn_pool_create(pool);
 704       int i;
 705
 706       for (i = 0; i < files_to_delete->nelts; ++i)
 707         {
 708           const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*);
 709
 710           svn_pool_clear(iterpool);
 711           SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
 712         }
 713
 714       svn_pool_destroy(iterpool);
 715     }
 716   return SVN_NO_ERROR;
 717 }
 718
 719 /* Write a pack file header to STREAM that starts at revision START_REVISION
 720  * and contains the indexes [START,END) of SIZES.
 721  */
 722 static svn_error_t *
 723 serialize_revprops_header(svn_stream_t *stream,
 724                           svn_revnum_t start_revision,
 725                           apr_array_header_t *sizes,
 726                           int start,
 727                           int end,
 728                           apr_pool_t *pool)
 729 {
 730   apr_pool_t *iterpool = svn_pool_create(pool);
 731   int i;
 732
 733   SVN_ERR_ASSERT(start < end);
 734
 735   /* start revision and entry count */
 736   SVN_ERR(svn_stream_printf(stream, pool, "%ld\n", start_revision));
 737   SVN_ERR(svn_stream_printf(stream, pool, "%d\n", end - start));
 738
 739   /* the sizes array */
 740   for (i = start; i < end; ++i)
 741     {
 742       /* Non-standard pool usage.
 743        *
 744        * We only allocate a few bytes each iteration -- even with a
 745        * million iterations we would still be in good shape memory-wise.
 746        */
 747       apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t);
 748       SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n",
 749                                 size));
 750     }
 751
 752   /* the double newline char indicates the end of the header */
 753   SVN_ERR(svn_stream_printf(stream, iterpool, "\n"));
 754
 755   svn_pool_destroy(iterpool);
 756   return SVN_NO_ERROR;
 757 }
 758
 759 /* Writes the a pack file to FILE.  It copies the serialized data
 760  * from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX.
 761  *
 762  * The data for the latter is taken from NEW_SERIALIZED.  Note, that
 763  * CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is
 764  * taken in that case but only a subset of the old data will be copied.
 765  *
 766  * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size.
 767  * POOL is used for temporary allocations.
 768  */
 769 static svn_error_t *
 770 repack_revprops(svn_fs_t *fs,
 771                 packed_revprops_t *revprops,
 772                 int start,
 773                 int end,
 774                 int changed_index,
 775                 svn_stringbuf_t *new_serialized,
 776                 apr_off_t new_total_size,
 777                 apr_file_t *file,
 778                 apr_pool_t *pool)
 779 {
 780   fs_fs_data_t *ffd = fs->fsap_data;
 781   svn_stream_t *stream;
 782   int i;
 783
 784   /* create data empty buffers and the stream object */
 785   svn_stringbuf_t *uncompressed
 786     = svn_stringbuf_create_ensure((apr_size_t)new_total_size, pool);
 787   svn_stringbuf_t *compressed
 788     = svn_stringbuf_create_empty(pool);
 789   stream = svn_stream_from_stringbuf(uncompressed, pool);
 790
 791   /* write the header*/
 792   SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start,
 793                                     revprops->sizes, start, end, pool));
 794
 795   /* append the serialized revprops */
 796   for (i = start; i < end; ++i)
 797     if (i == changed_index)
 798       {
 799         SVN_ERR(svn_stream_write(stream,
 800                                  new_serialized->data,
 801                                  &new_serialized->len));
 802       }
 803     else
 804       {
 805         apr_size_t size
 806             = (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t);
 807         apr_size_t offset
 808             = (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t);
 809
 810         SVN_ERR(svn_stream_write(stream,
 811                                  revprops->packed_revprops->data + offset,
 812                                  &size));
 813       }
 814
 815   /* flush the stream buffer (if any) to our underlying data buffer */
 816   SVN_ERR(svn_stream_close(stream));
 817
 818   /* compress / store the data */
 819   SVN_ERR(svn__compress(uncompressed,
 820                         compressed,
 821                         ffd->compress_packed_revprops
 822                           ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
 823                           : SVN_DELTA_COMPRESSION_LEVEL_NONE));
 824
 825   /* finally, write the content to the target file, flush and close it */
 826   SVN_ERR(svn_io_file_write_full(file, compressed->data, compressed->len,
 827                                  NULL, pool));
 828   SVN_ERR(svn_io_file_flush_to_disk(file, pool));
 829   SVN_ERR(svn_io_file_close(file, pool));
 830
 831   return SVN_NO_ERROR;
 832 }
 833
 834 /* Allocate a new pack file name for revisions
 835  *     [REVPROPS->START_REVISION + START, REVPROPS->START_REVISION + END - 1]
 836  * of REVPROPS->MANIFEST.  Add the name of old file to FILES_TO_DELETE,
 837  * auto-create that array if necessary.  Return an open file *FILE that is
 838  * allocated in POOL.
 839  */
 840 static svn_error_t *
 841 repack_file_open(apr_file_t **file,
 842                  svn_fs_t *fs,
 843                  packed_revprops_t *revprops,
 844                  int start,
 845                  int end,
 846                  apr_array_header_t **files_to_delete,
 847                  apr_pool_t *pool)
 848 {
 849   apr_int64_t tag;
 850   const char *tag_string;
 851   svn_string_t *new_filename;
 852   int i;
 853   int manifest_offset
 854     = (int)(revprops->start_revision - revprops->manifest_start);
 855
 856   /* get the old (= current) file name and enlist it for later deletion */
 857   const char *old_filename = APR_ARRAY_IDX(revprops->manifest,
 858                                            start + manifest_offset,
 859                                            const char*);
 860
 861   if (*files_to_delete == NULL)
 862     *files_to_delete = apr_array_make(pool, 3, sizeof(const char*));
 863
 864   APR_ARRAY_PUSH(*files_to_delete, const char*)
 865     = svn_dirent_join(revprops->folder, old_filename, pool);
 866
 867   /* increase the tag part, i.e. the counter after the dot */
 868   tag_string = strchr(old_filename, '.');
 869   if (tag_string == NULL)
 870     return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
 871                              _("Packed file '%s' misses a tag"),
 872                              old_filename);
 873
 874   SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1));
 875   new_filename = svn_string_createf(pool, "%ld.%" APR_INT64_T_FMT,
 876                                     revprops->start_revision + start,
 877                                     ++tag);
 878
 879   /* update the manifest to point to the new file */
 880   for (i = start; i < end; ++i)
 881     APR_ARRAY_IDX(revprops->manifest, i + manifest_offset, const char*)
 882       = new_filename->data;
 883
 884   /* open the file */
 885   SVN_ERR(svn_io_file_open(file, svn_dirent_join(revprops->folder,
 886                                                  new_filename->data,
 887                                                  pool),
 888                            APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool));
 889
 890   return SVN_NO_ERROR;
 891 }
 892
 893 /* For revision REV in filesystem FS, set the revision properties to
 894  * PROPLIST.  Return a new file in *TMP_PATH that the caller shall move
 895  * to *FINAL_PATH to make the change visible.  Files to be deleted will
 896  * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated.
 897  * Use POOL for allocations.
 898  */
 899 static svn_error_t *
 900 write_packed_revprop(const char **final_path,
 901                      const char **tmp_path,
 902                      apr_array_header_t **files_to_delete,
 903                      svn_fs_t *fs,
 904                      svn_revnum_t rev,
 905                      apr_hash_t *proplist,
 906                      apr_pool_t *pool)
 907 {
 908   fs_fs_data_t *ffd = fs->fsap_data;
 909   packed_revprops_t *revprops;
 910   apr_int64_t generation = 0;
 911   svn_stream_t *stream;
 912   apr_file_t *file;
 913   svn_stringbuf_t *serialized;
 914   apr_off_t new_total_size;
 915   int changed_index;
 916
 917   /* read contents of the current pack file */
 918   SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, TRUE, pool));
 919
 920   /* serialize the new revprops */
 921   serialized = svn_stringbuf_create_empty(pool);
 922   stream = svn_stream_from_stringbuf(serialized, pool);
 923   SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
 924   SVN_ERR(svn_stream_close(stream));
 925
 926   /* calculate the size of the new data */
 927   changed_index = (int)(rev - revprops->start_revision);
 928   new_total_size = revprops->total_size - revprops->serialized_size
 929                  + serialized->len
 930                  + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE;
 931
 932   APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len;
 933
 934   /* can we put the new data into the same pack as the before? */
 935   if (   new_total_size < ffd->revprop_pack_size
 936       || revprops->sizes->nelts == 1)
 937     {
 938       /* simply replace the old pack file with new content as we do it
 939        * in the non-packed case */
 940
 941       *final_path = svn_dirent_join(revprops->folder, revprops->filename,
 942                                     pool);
 943       SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, revprops->folder,
 944                                        svn_io_file_del_none, pool, pool));
 945       SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts,
 946                               changed_index, serialized, new_total_size,
 947                               file, pool));
 948     }
 949   else
 950     {
 951       /* split the pack file into two of roughly equal size */
 952       int right_count, left_count, i;
 953
 954       int left = 0;
 955       int right = revprops->sizes->nelts - 1;
 956       apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE;
 957       apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE;
 958
 959       /* let left and right side grow such that their size difference
 960        * is minimal after each step. */
 961       while (left <= right)
 962         if (  left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t)
 963             < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t))
 964           {
 965             left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t)
 966                       + SVN_INT64_BUFFER_SIZE;
 967             ++left;
 968           }
 969         else
 970           {
 971             right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)
 972                         + SVN_INT64_BUFFER_SIZE;
 973             --right;
 974           }
 975
 976        /* since the items need much less than SVN_INT64_BUFFER_SIZE
 977         * bytes to represent their length, the split may not be optimal */
 978       left_count = left;
 979       right_count = revprops->sizes->nelts - left;
 980
 981       /* if new_size is large, one side may exceed the pack size limit.
 982        * In that case, split before and after the modified revprop.*/
 983       if (   left_size > ffd->revprop_pack_size
 984           || right_size > ffd->revprop_pack_size)
 985         {
 986           left_count = changed_index;
 987           right_count = revprops->sizes->nelts - left_count - 1;
 988         }
 989
 990       /* write the new, split files */
 991       if (left_count)
 992         {
 993           SVN_ERR(repack_file_open(&file, fs, revprops, 0,
 994                                    left_count, files_to_delete, pool));
 995           SVN_ERR(repack_revprops(fs, revprops, 0, left_count,
 996                                   changed_index, serialized, new_total_size,
 997                                   file, pool));
 998         }
 999
1000       if (left_count + right_count < revprops->sizes->nelts)
1001         {
1002           SVN_ERR(repack_file_open(&file, fs, revprops, changed_index,
1003                                    changed_index + 1, files_to_delete,
1004                                    pool));
1005           SVN_ERR(repack_revprops(fs, revprops, changed_index,
1006                                   changed_index + 1,
1007                                   changed_index, serialized, new_total_size,
1008                                   file, pool));
1009         }
1010
1011       if (right_count)
1012         {
1013           SVN_ERR(repack_file_open(&file, fs, revprops,
1014                                    revprops->sizes->nelts - right_count,
1015                                    revprops->sizes->nelts,
1016                                    files_to_delete, pool));
1017           SVN_ERR(repack_revprops(fs, revprops,
1018                                   revprops->sizes->nelts - right_count,
1019                                   revprops->sizes->nelts, changed_index,
1020                                   serialized, new_total_size, file,
1021                                   pool));
1022         }
1023
1024       /* write the new manifest */
1025       *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool);
1026       SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, revprops->folder,
1027                                        svn_io_file_del_none, pool, pool));
1028
1029       for (i = 0; i < revprops->manifest->nelts; ++i)
1030         {
1031           const char *filename = APR_ARRAY_IDX(revprops->manifest, i,
1032                                                const char*);
1033           SVN_ERR(svn_io_file_write_full(file, filename, strlen(filename),
1034                                          NULL, pool));
1035           SVN_ERR(svn_io_file_putc('\n', file, pool));
1036         }
1037
1038       SVN_ERR(svn_io_file_flush_to_disk(file, pool));
1039       SVN_ERR(svn_io_file_close(file, pool));
1040     }
1041
1042   return SVN_NO_ERROR;
1043 }
1044
1045 /* Set the revision property list of revision REV in filesystem FS to
1046    PROPLIST.  Use POOL for temporary allocations. */
1047 svn_error_t *
1048 svn_fs_fs__set_revision_proplist(svn_fs_t *fs,
1049                                  svn_revnum_t rev,
1050                                  apr_hash_t *proplist,
1051                                  apr_pool_t *pool)
1052 {
1053   svn_boolean_t is_packed;
1054   const char *final_path;
1055   const char *tmp_path;
1056   const char *perms_reference;
1057   apr_array_header_t *files_to_delete = NULL;
1058
1059   SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, pool));
1060
1061   /* this info will not change while we hold the global FS write lock */
1062   is_packed = svn_fs_fs__is_packed_revprop(fs, rev);
1063
1064   /* Serialize the new revprop data */
1065   if (is_packed)
1066     SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete,
1067                                  fs, rev, proplist, pool));
1068   else
1069     SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path,
1070                                      fs, rev, proplist, pool));
1071
1072   /* We use the rev file of this revision as the perms reference,
1073    * because when setting revprops for the first time, the revprop
1074    * file won't exist and therefore can't serve as its own reference.
1075    * (Whereas the rev file should already exist at this point.)
1076    */
1077   perms_reference = svn_fs_fs__path_rev_absolute(fs, rev, pool);
1078
1079   /* Now, switch to the new revprop data. */
1080   SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference,
1081                                 files_to_delete, pool));
1082
1083   return SVN_NO_ERROR;
1084 }
1085
1086 /* Return TRUE, if for REVISION in FS, we can find the revprop pack file.
1087  * Use POOL for temporary allocations.
1088  * Set *MISSING, if the reason is a missing manifest or pack file.
1089  */
1090 svn_boolean_t
1091 svn_fs_fs__packed_revprop_available(svn_boolean_t *missing,
1092                                     svn_fs_t *fs,
1093                                     svn_revnum_t revision,
1094                                     apr_pool_t *pool)
1095 {
1096   fs_fs_data_t *ffd = fs->fsap_data;
1097   svn_stringbuf_t *content = NULL;
1098
1099   /* try to read the manifest file */
1100   const char *folder
1101     = svn_fs_fs__path_revprops_pack_shard(fs, revision, pool);
1102   const char *manifest_path = svn_dirent_join(folder, PATH_MANIFEST, pool);
1103
1104   svn_error_t *err = svn_fs_fs__try_stringbuf_from_file(&content,
1105                                                         missing,
1106                                                         manifest_path,
1107                                                         FALSE,
1108                                                         pool);
1109
1110   /* if the manifest cannot be read, consider the pack files inaccessible
1111    * even if the file itself exists. */
1112   if (err)
1113     {
1114       svn_error_clear(err);
1115       return FALSE;
1116     }
1117
1118   if (*missing)
1119     return FALSE;
1120
1121   /* parse manifest content until we find the entry for REVISION.
1122    * Revision 0 is never packed. */
1123   revision = revision < ffd->max_files_per_dir
1124            ? revision - 1
1125            : revision % ffd->max_files_per_dir;
1126   while (content->data)
1127     {
1128       char *next = strchr(content->data, '\n');
1129       if (next)
1130         {
1131           *next = 0;
1132           ++next;
1133         }
1134
1135       if (revision-- == 0)
1136         {
1137           /* the respective pack file must exist (and be a file) */
1138           svn_node_kind_t kind;
1139           err = svn_io_check_path(svn_dirent_join(folder, content->data,
1140                                                   pool),
1141                                   &kind, pool);
1142           if (err)
1143             {
1144               svn_error_clear(err);
1145               return FALSE;
1146             }
1147
1148           *missing = kind == svn_node_none;
1149           return kind == svn_node_file;
1150         }
1151
1152       content->data = next;
1153     }
1154
1155   return FALSE;
1156 }
1157
1158 \f
1159 /****** Packing FSFS shards *********/
1160
1161 svn_error_t *
1162 svn_fs_fs__copy_revprops(const char *pack_file_dir,
1163                          const char *pack_filename,
1164                          const char *shard_path,
1165                          svn_revnum_t start_rev,
1166                          svn_revnum_t end_rev,
1167                          apr_array_header_t *sizes,
1168                          apr_size_t total_size,
1169                          int compression_level,
1170                          svn_cancel_func_t cancel_func,
1171                          void *cancel_baton,
1172                          apr_pool_t *scratch_pool)
1173 {
1174   svn_stream_t *pack_stream;
1175   apr_file_t *pack_file;
1176   svn_revnum_t rev;
1177   apr_pool_t *iterpool = svn_pool_create(scratch_pool);
1178
1179   /* create empty data buffer and a write stream on top of it */
1180   svn_stringbuf_t *uncompressed
1181     = svn_stringbuf_create_ensure(total_size, scratch_pool);
1182   svn_stringbuf_t *compressed
1183     = svn_stringbuf_create_empty(scratch_pool);
1184   pack_stream = svn_stream_from_stringbuf(uncompressed, scratch_pool);
1185
1186   /* write the pack file header */
1187   SVN_ERR(serialize_revprops_header(pack_stream, start_rev, sizes, 0,
1188                                     sizes->nelts, iterpool));
1189
1190   /* Some useful paths. */
1191   SVN_ERR(svn_io_file_open(&pack_file, svn_dirent_join(pack_file_dir,
1192                                                        pack_filename,
1193                                                        scratch_pool),
1194                            APR_WRITE | APR_CREATE, APR_OS_DEFAULT,
1195                            scratch_pool));
1196
1197   /* Iterate over the revisions in this shard, squashing them together. */
1198   for (rev = start_rev; rev <= end_rev; rev++)
1199     {
1200       const char *path;
1201       svn_stream_t *stream;
1202
1203       svn_pool_clear(iterpool);
1204
1205       /* Construct the file name. */
1206       path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
1207                              iterpool);
1208
1209       /* Copy all the bits from the non-packed revprop file to the end of
1210        * the pack file. */
1211       SVN_ERR(svn_stream_open_readonly(&stream, path, iterpool, iterpool));
1212       SVN_ERR(svn_stream_copy3(stream, pack_stream,
1213                                cancel_func, cancel_baton, iterpool));
1214     }
1215
1216   /* flush stream buffers to content buffer */
1217   SVN_ERR(svn_stream_close(pack_stream));
1218
1219   /* compress the content (or just store it for COMPRESSION_LEVEL 0) */
1220   SVN_ERR(svn__compress(uncompressed, compressed, compression_level));
1221
1222   /* write the pack file content to disk */
1223   SVN_ERR(svn_io_file_write_full(pack_file, compressed->data, compressed->len,
1224                                  NULL, scratch_pool));
1225   SVN_ERR(svn_io_file_flush_to_disk(pack_file, scratch_pool));
1226   SVN_ERR(svn_io_file_close(pack_file, scratch_pool));
1227
1228   svn_pool_destroy(iterpool);
1229
1230   return SVN_NO_ERROR;
1231 }
1232
1233 svn_error_t *
1234 svn_fs_fs__pack_revprops_shard(const char *pack_file_dir,
1235                                const char *shard_path,
1236                                apr_int64_t shard,
1237                                int max_files_per_dir,
1238                                apr_off_t max_pack_size,
1239                                int compression_level,
1240                                svn_cancel_func_t cancel_func,
1241                                void *cancel_baton,
1242                                apr_pool_t *scratch_pool)
1243 {
1244   const char *manifest_file_path, *pack_filename = NULL;
1245   apr_file_t *manifest_file;
1246   svn_stream_t *manifest_stream;
1247   svn_revnum_t start_rev, end_rev, rev;
1248   apr_off_t total_size;
1249   apr_pool_t *iterpool = svn_pool_create(scratch_pool);
1250   apr_array_header_t *sizes;
1251
1252   /* Some useful paths. */
1253   manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST,
1254                                        scratch_pool);
1255
1256   /* Remove any existing pack file for this shard, since it is incomplete. */
1257   SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton,
1258                              scratch_pool));
1259
1260   /* Create the new directory and manifest file stream. */
1261   SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool));
1262
1263   SVN_ERR(svn_io_file_open(&manifest_file, manifest_file_path,
1264                            APR_WRITE | APR_BUFFERED | APR_CREATE | APR_EXCL,
1265                            APR_OS_DEFAULT, scratch_pool));
1266   manifest_stream = svn_stream_from_aprfile2(manifest_file, TRUE,
1267                                              scratch_pool);
1268
1269   /* revisions to handle. Special case: revision 0 */
1270   start_rev = (svn_revnum_t) (shard * max_files_per_dir);
1271   end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1);
1272   if (start_rev == 0)
1273     ++start_rev;
1274     /* Special special case: if max_files_per_dir is 1, then at this point
1275        start_rev == 1 and end_rev == 0 (!).  Fortunately, everything just
1276        works. */
1277
1278   /* initialize the revprop size info */
1279   sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_off_t));
1280   total_size = 2 * SVN_INT64_BUFFER_SIZE;
1281
1282   /* Iterate over the revisions in this shard, determine their size and
1283    * squashing them together into pack files. */
1284   for (rev = start_rev; rev <= end_rev; rev++)
1285     {
1286       apr_finfo_t finfo;
1287       const char *path;
1288
1289       svn_pool_clear(iterpool);
1290
1291       /* Get the size of the file. */
1292       path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
1293                              iterpool);
1294       SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool));
1295
1296       /* if we already have started a pack file and this revprop cannot be
1297        * appended to it, write the previous pack file. */
1298       if (sizes->nelts != 0 &&
1299           total_size + SVN_INT64_BUFFER_SIZE + finfo.size > max_pack_size)
1300         {
1301           SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename,
1302                                            shard_path, start_rev, rev-1,
1303                                            sizes, (apr_size_t)total_size,
1304                                            compression_level, cancel_func,
1305                                            cancel_baton, iterpool));
1306
1307           /* next pack file starts empty again */
1308           apr_array_clear(sizes);
1309           total_size = 2 * SVN_INT64_BUFFER_SIZE;
1310           start_rev = rev;
1311         }
1312
1313       /* Update the manifest. Allocate a file name for the current pack
1314        * file if it is a new one */
1315       if (sizes->nelts == 0)
1316         pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev);
1317
1318       SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%s\n",
1319                                 pack_filename));
1320
1321       /* add to list of files to put into the current pack file */
1322       APR_ARRAY_PUSH(sizes, apr_off_t) = finfo.size;
1323       total_size += SVN_INT64_BUFFER_SIZE + finfo.size;
1324     }
1325
1326   /* write the last pack file */
1327   if (sizes->nelts != 0)
1328     SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename,
1329                                      shard_path, start_rev, rev-1,
1330                                      sizes, (apr_size_t)total_size,
1331                                      compression_level, cancel_func,
1332                                      cancel_baton, iterpool));
1333
1334   /* flush the manifest file to disk and update permissions */
1335   SVN_ERR(svn_stream_close(manifest_stream));
1336   SVN_ERR(svn_io_file_flush_to_disk(manifest_file, iterpool));
1337   SVN_ERR(svn_io_file_close(manifest_file, iterpool));
1338   SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool));
1339
1340   svn_pool_destroy(iterpool);
1341
1342   return SVN_NO_ERROR;
1343 }
1344
1345 svn_error_t *
1346 svn_fs_fs__delete_revprops_shard(const char *shard_path,
1347                                  apr_int64_t shard,
1348                                  int max_files_per_dir,
1349                                  svn_cancel_func_t cancel_func,
1350                                  void *cancel_baton,
1351                                  apr_pool_t *scratch_pool)
1352 {
1353   if (shard == 0)
1354     {
1355       apr_pool_t *iterpool = svn_pool_create(scratch_pool);
1356       int i;
1357
1358       /* delete all files except the one for revision 0 */
1359       for (i = 1; i < max_files_per_dir; ++i)
1360         {
1361           const char *path;
1362           svn_pool_clear(iterpool);
1363
1364           path = svn_dirent_join(shard_path,
1365                                  apr_psprintf(iterpool, "%d", i),
1366                                  iterpool);
1367           if (cancel_func)
1368             SVN_ERR((*cancel_func)(cancel_baton));
1369
1370           SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
1371         }
1372
1373       svn_pool_destroy(iterpool);
1374     }
1375   else
1376     SVN_ERR(svn_io_remove_dir2(shard_path, TRUE,
1377                                cancel_func, cancel_baton, scratch_pool));
1378
1379   return SVN_NO_ERROR;
1380 }
1381