2 * ====================================================================
3 * Licensed to the Apache Software Foundation (ASF) under one
4 * or more contributor license agreements. See the NOTICE file
5 * distributed with this work for additional information
6 * regarding copyright ownership. The ASF licenses this file
7 * to you under the Apache License, Version 2.0 (the
8 * "License"); you may not use this file except in compliance
9 * with the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing,
14 * software distributed under the License is distributed on an
15 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 * KIND, either express or implied. See the License for the
17 * specific language governing permissions and limitations
19 * ====================================================================
23 #include "svn_cmdline.h"
24 #include "svn_config.h"
25 #include "svn_pools.h"
26 #include "svn_delta.h"
27 #include "svn_dirent_uri.h"
29 #include "svn_props.h"
34 #include "svn_subst.h"
35 #include "svn_string.h"
37 #include "private/svn_string_private.h"
41 #include "svn_private_config.h"
43 #include <apr_network_io.h>
44 #include <apr_signal.h>
48 /* Normalize the encoding and line ending style of *STR, so that it contains
49 * only LF (\n) line endings and is encoded in UTF-8. After return, *STR may
50 * point at a new svn_string_t* allocated in RESULT_POOL.
52 * If SOURCE_PROP_ENCODING is NULL, then *STR is presumed to be encoded in
55 * *WAS_NORMALIZED is set to TRUE when *STR needed line ending normalization.
56 * Otherwise it is set to FALSE.
58 * SCRATCH_POOL is used for temporary allocations.
61 normalize_string(const svn_string_t **str,
62 svn_boolean_t *was_normalized,
63 const char *source_prop_encoding,
64 apr_pool_t *result_pool,
65 apr_pool_t *scratch_pool)
67 svn_string_t *new_str;
69 *was_normalized = FALSE;
74 SVN_ERR_ASSERT((*str)->data != NULL);
76 if (source_prop_encoding == NULL)
77 source_prop_encoding = "UTF-8";
80 SVN_ERR(svn_subst_translate_string2(&new_str, NULL, was_normalized,
81 *str, source_prop_encoding, TRUE,
82 result_pool, scratch_pool));
88 /* Remove r0 references from the mergeinfo string *STR.
90 * r0 was never a valid mergeinfo reference and cannot be committed with
91 * recent servers, but can be committed through a server older than 1.6.18
92 * for HTTP or older than 1.6.17 for the other protocols. See issue #4476
93 * "Mergeinfo containing r0 makes svnsync and dump and load fail".
95 * Set *WAS_CHANGED to TRUE if *STR was changed, otherwise to FALSE.
98 remove_r0_mergeinfo(const svn_string_t **str,
99 svn_boolean_t *was_changed,
100 apr_pool_t *result_pool,
101 apr_pool_t *scratch_pool)
103 svn_stringbuf_t *new_str = svn_stringbuf_create_empty(result_pool);
104 apr_array_header_t *lines;
107 SVN_ERR_ASSERT(*str && (*str)->data);
109 *was_changed = FALSE;
112 lines = svn_cstring_split((*str)->data, "\n", FALSE, scratch_pool);
114 for (i = 0; i < lines->nelts; i++)
116 char *line = APR_ARRAY_IDX(lines, i, char *);
120 /* split at the last colon */
121 colon = strrchr(line, ':');
124 return svn_error_createf(SVN_ERR_MERGEINFO_PARSE_ERROR, NULL,
125 _("Missing colon in svn:mergeinfo "
128 rangelist = colon + 1;
133 if (strncmp(rangelist, "0*,", 3) == 0)
137 else if (strcmp(rangelist, "0*") == 0
138 || strncmp(rangelist, "0,", 2) == 0
139 || strncmp(rangelist, "0-1*", 4) == 0
140 || strncmp(rangelist, "0-1,", 4) == 0
141 || strcmp(rangelist, "0-1") == 0)
145 else if (strcmp(rangelist, "0") == 0)
149 else if (strncmp(rangelist, "0-", 2) == 0)
159 svn_stringbuf_appendbyte(new_str, '\n');
160 svn_stringbuf_appendbytes(new_str, line, colon + 1 - line);
161 svn_stringbuf_appendcstr(new_str, rangelist);
165 if (strcmp((*str)->data, new_str->data) != 0)
170 *str = svn_stringbuf__morph_into_string(new_str);
175 /* Normalize the encoding and line ending style of the values of properties
176 * in REV_PROPS that "need translation" (according to
177 * svn_prop_needs_translation(), which is currently all svn:* props) so that
178 * they are encoded in UTF-8 and contain only LF (\n) line endings.
180 * The number of properties that needed line ending normalization is returned in
183 * No re-encoding is performed if SOURCE_PROP_ENCODING is NULL.
186 svnsync_normalize_revprops(apr_hash_t *rev_props,
187 int *normalized_count,
188 const char *source_prop_encoding,
191 apr_hash_index_t *hi;
192 *normalized_count = 0;
194 for (hi = apr_hash_first(pool, rev_props);
196 hi = apr_hash_next(hi))
198 const char *propname = apr_hash_this_key(hi);
199 const svn_string_t *propval = apr_hash_this_val(hi);
201 if (svn_prop_needs_translation(propname))
203 svn_boolean_t was_normalized;
204 SVN_ERR(normalize_string(&propval, &was_normalized,
205 source_prop_encoding, pool, pool));
207 /* Replace the existing prop value. */
208 svn_hash_sets(rev_props, propname, propval);
211 (*normalized_count)++; /* Count it. */
218 /*** Synchronization Editor ***/
220 /* This editor has a couple of jobs.
222 * First, it needs to filter out the propchanges that can't be passed over
225 * Second, it needs to adjust for the fact that we might not actually have
226 * permission to see all of the data from the remote repository, which means
227 * we could get revisions that are totally empty from our point of view.
229 * Third, it needs to adjust copyfrom paths, adding the root url for the
230 * destination repository to the beginning of them.
235 typedef struct edit_baton_t {
236 const svn_delta_editor_t *wrapped_editor;
237 void *wrapped_edit_baton;
238 const char *to_url; /* URL we're copying into, for correct copyfrom URLs */
239 const char *source_prop_encoding;
240 svn_boolean_t called_open_root;
241 svn_boolean_t got_textdeltas;
242 svn_revnum_t base_revision;
244 svn_boolean_t mergeinfo_tweaked; /* Did we tweak svn:mergeinfo? */
245 svn_boolean_t strip_mergeinfo; /* Are we stripping svn:mergeinfo? */
246 svn_boolean_t migrate_svnmerge; /* Are we converting svnmerge.py data? */
247 svn_boolean_t mergeinfo_stripped; /* Did we strip svn:mergeinfo? */
248 svn_boolean_t svnmerge_migrated; /* Did we convert svnmerge.py data? */
249 svn_boolean_t svnmerge_blocked; /* Was there any blocked svnmerge data? */
250 int *normalized_node_props_counter; /* Where to count normalizations? */
254 /* A dual-purpose baton for files and directories. */
255 typedef struct node_baton_t {
257 void *wrapped_node_baton;
261 /*** Editor vtable functions ***/
264 set_target_revision(void *edit_baton,
265 svn_revnum_t target_revision,
268 edit_baton_t *eb = edit_baton;
269 return eb->wrapped_editor->set_target_revision(eb->wrapped_edit_baton,
270 target_revision, pool);
274 open_root(void *edit_baton,
275 svn_revnum_t base_revision,
279 edit_baton_t *eb = edit_baton;
280 node_baton_t *dir_baton = apr_palloc(pool, sizeof(*dir_baton));
282 SVN_ERR(eb->wrapped_editor->open_root(eb->wrapped_edit_baton,
284 &dir_baton->wrapped_node_baton));
286 eb->called_open_root = TRUE;
287 dir_baton->edit_baton = edit_baton;
288 *root_baton = dir_baton;
294 delete_entry(const char *path,
295 svn_revnum_t base_revision,
299 node_baton_t *pb = parent_baton;
300 edit_baton_t *eb = pb->edit_baton;
302 return eb->wrapped_editor->delete_entry(path, base_revision,
303 pb->wrapped_node_baton, pool);
307 add_directory(const char *path,
309 const char *copyfrom_path,
310 svn_revnum_t copyfrom_rev,
314 node_baton_t *pb = parent_baton;
315 edit_baton_t *eb = pb->edit_baton;
316 node_baton_t *b = apr_palloc(pool, sizeof(*b));
318 /* if copyfrom_path is an fspath create a proper uri */
319 if (copyfrom_path && copyfrom_path[0] == '/')
320 copyfrom_path = svn_path_url_add_component2(eb->to_url,
321 copyfrom_path + 1, pool);
323 SVN_ERR(eb->wrapped_editor->add_directory(path, pb->wrapped_node_baton,
326 &b->wrapped_node_baton));
335 open_directory(const char *path,
337 svn_revnum_t base_revision,
341 node_baton_t *pb = parent_baton;
342 edit_baton_t *eb = pb->edit_baton;
343 node_baton_t *db = apr_palloc(pool, sizeof(*db));
345 SVN_ERR(eb->wrapped_editor->open_directory(path, pb->wrapped_node_baton,
347 &db->wrapped_node_baton));
356 add_file(const char *path,
358 const char *copyfrom_path,
359 svn_revnum_t copyfrom_rev,
363 node_baton_t *pb = parent_baton;
364 edit_baton_t *eb = pb->edit_baton;
365 node_baton_t *fb = apr_palloc(pool, sizeof(*fb));
367 /* if copyfrom_path is an fspath create a proper uri */
368 if (copyfrom_path && copyfrom_path[0] == '/')
369 copyfrom_path = svn_path_url_add_component2(eb->to_url,
370 copyfrom_path + 1, pool);
372 SVN_ERR(eb->wrapped_editor->add_file(path, pb->wrapped_node_baton,
373 copyfrom_path, copyfrom_rev,
374 pool, &fb->wrapped_node_baton));
383 open_file(const char *path,
385 svn_revnum_t base_revision,
389 node_baton_t *pb = parent_baton;
390 edit_baton_t *eb = pb->edit_baton;
391 node_baton_t *fb = apr_palloc(pool, sizeof(*fb));
393 SVN_ERR(eb->wrapped_editor->open_file(path, pb->wrapped_node_baton,
395 &fb->wrapped_node_baton));
404 apply_textdelta(void *file_baton,
405 const char *base_checksum,
407 svn_txdelta_window_handler_t *handler,
408 void **handler_baton)
410 node_baton_t *fb = file_baton;
411 edit_baton_t *eb = fb->edit_baton;
415 if (! eb->got_textdeltas)
416 SVN_ERR(svn_cmdline_printf(pool, _("Transmitting file data ")));
417 SVN_ERR(svn_cmdline_printf(pool, "."));
418 SVN_ERR(svn_cmdline_fflush(stdout));
421 eb->got_textdeltas = TRUE;
422 return eb->wrapped_editor->apply_textdelta(fb->wrapped_node_baton,
424 handler, handler_baton);
428 close_file(void *file_baton,
429 const char *text_checksum,
432 node_baton_t *fb = file_baton;
433 edit_baton_t *eb = fb->edit_baton;
434 return eb->wrapped_editor->close_file(fb->wrapped_node_baton,
435 text_checksum, pool);
439 absent_file(const char *path,
443 node_baton_t *fb = file_baton;
444 edit_baton_t *eb = fb->edit_baton;
445 return eb->wrapped_editor->absent_file(path, fb->wrapped_node_baton, pool);
449 close_directory(void *dir_baton,
452 node_baton_t *db = dir_baton;
453 edit_baton_t *eb = db->edit_baton;
454 return eb->wrapped_editor->close_directory(db->wrapped_node_baton, pool);
458 absent_directory(const char *path,
462 node_baton_t *db = dir_baton;
463 edit_baton_t *eb = db->edit_baton;
464 return eb->wrapped_editor->absent_directory(path, db->wrapped_node_baton,
469 change_file_prop(void *file_baton,
471 const svn_string_t *value,
474 node_baton_t *fb = file_baton;
475 edit_baton_t *eb = fb->edit_baton;
477 /* only regular properties can pass over libsvn_ra */
478 if (svn_property_kind2(name) != svn_prop_regular_kind)
481 /* Maybe drop svn:mergeinfo. */
482 if (eb->strip_mergeinfo && (strcmp(name, SVN_PROP_MERGEINFO) == 0))
484 eb->mergeinfo_stripped = TRUE;
488 /* Maybe drop (errantly set, as this is a file) svnmerge.py properties. */
489 if (eb->migrate_svnmerge && (strcmp(name, "svnmerge-integrated") == 0))
491 eb->svnmerge_migrated = TRUE;
495 /* Remember if we see any svnmerge-blocked properties. (They really
496 shouldn't be here, as this is a file, but whatever...) */
497 if (eb->migrate_svnmerge && (strcmp(name, "svnmerge-blocked") == 0))
499 eb->svnmerge_blocked = TRUE;
502 /* Normalize svn:* properties as necessary. */
503 if (svn_prop_needs_translation(name))
505 svn_boolean_t was_normalized;
506 svn_boolean_t mergeinfo_tweaked = FALSE;
508 /* Normalize encoding to UTF-8, and EOL style to LF. */
509 SVN_ERR(normalize_string(&value, &was_normalized,
510 eb->source_prop_encoding, pool, pool));
511 /* Correct malformed mergeinfo. */
512 if (value && strcmp(name, SVN_PROP_MERGEINFO) == 0)
514 SVN_ERR(remove_r0_mergeinfo(&value, &mergeinfo_tweaked,
516 if (mergeinfo_tweaked)
517 eb->mergeinfo_tweaked = TRUE;
520 (*(eb->normalized_node_props_counter))++;
523 return eb->wrapped_editor->change_file_prop(fb->wrapped_node_baton,
528 change_dir_prop(void *dir_baton,
530 const svn_string_t *value,
533 node_baton_t *db = dir_baton;
534 edit_baton_t *eb = db->edit_baton;
536 /* Only regular properties can pass over libsvn_ra */
537 if (svn_property_kind2(name) != svn_prop_regular_kind)
540 /* Maybe drop svn:mergeinfo. */
541 if (eb->strip_mergeinfo && (strcmp(name, SVN_PROP_MERGEINFO) == 0))
543 eb->mergeinfo_stripped = TRUE;
547 /* Maybe convert svnmerge-integrated data into svn:mergeinfo. (We
548 ignore svnmerge-blocked for now.) */
549 /* ### FIXME: Consult the mirror repository's HEAD prop values and
550 ### merge svn:mergeinfo, svnmerge-integrated, and svnmerge-blocked. */
551 if (eb->migrate_svnmerge && (strcmp(name, "svnmerge-integrated") == 0))
555 /* svnmerge-integrated differs from svn:mergeinfo in a pair
556 of ways. First, it can use tabs, newlines, or spaces to
557 delimit source information. Secondly, the source paths
558 are relative URLs, whereas svn:mergeinfo uses relative
559 paths (not URI-encoded). */
561 svn_stringbuf_t *mergeinfo_buf = svn_stringbuf_create_empty(pool);
562 svn_mergeinfo_t mergeinfo;
564 apr_array_header_t *sources =
565 svn_cstring_split(value->data, " \t\n", TRUE, pool);
566 svn_string_t *new_value;
568 for (i = 0; i < sources->nelts; i++)
570 const char *rel_path;
571 apr_array_header_t *path_revs =
572 svn_cstring_split(APR_ARRAY_IDX(sources, i, const char *),
575 /* ### TODO: Warn? */
576 if (path_revs->nelts != 2)
579 /* Append this source's mergeinfo data. */
580 rel_path = APR_ARRAY_IDX(path_revs, 0, const char *);
581 rel_path = svn_path_uri_decode(rel_path, pool);
582 svn_stringbuf_appendcstr(mergeinfo_buf, rel_path);
583 svn_stringbuf_appendcstr(mergeinfo_buf, ":");
584 svn_stringbuf_appendcstr(mergeinfo_buf,
585 APR_ARRAY_IDX(path_revs, 1,
587 svn_stringbuf_appendcstr(mergeinfo_buf, "\n");
590 /* Try to parse the mergeinfo string we've created, just to
591 check for bogosity. If all goes well, we'll unparse it
592 again and use that as our property value. */
593 err = svn_mergeinfo_parse(&mergeinfo, mergeinfo_buf->data, pool);
596 svn_error_clear(err);
599 SVN_ERR(svn_mergeinfo_to_string(&new_value, mergeinfo, pool));
602 name = SVN_PROP_MERGEINFO;
603 eb->svnmerge_migrated = TRUE;
606 /* Remember if we see any svnmerge-blocked properties. */
607 if (eb->migrate_svnmerge && (strcmp(name, "svnmerge-blocked") == 0))
609 eb->svnmerge_blocked = TRUE;
612 /* Normalize svn:* properties as necessary. */
613 if (svn_prop_needs_translation(name))
615 svn_boolean_t was_normalized;
616 svn_boolean_t mergeinfo_tweaked = FALSE;
618 /* Normalize encoding to UTF-8, and EOL style to LF. */
619 SVN_ERR(normalize_string(&value, &was_normalized, eb->source_prop_encoding,
621 /* Maybe adjust svn:mergeinfo. */
622 if (value && strcmp(name, SVN_PROP_MERGEINFO) == 0)
624 SVN_ERR(remove_r0_mergeinfo(&value, &mergeinfo_tweaked,
626 if (mergeinfo_tweaked)
627 eb->mergeinfo_tweaked = TRUE;
630 (*(eb->normalized_node_props_counter))++;
633 return eb->wrapped_editor->change_dir_prop(db->wrapped_node_baton,
638 close_edit(void *edit_baton,
641 edit_baton_t *eb = edit_baton;
643 /* If we haven't opened the root yet, that means we're transfering
644 an empty revision, probably because we aren't allowed to see the
645 contents for some reason. In any event, we need to open the root
646 and close it again, before we can close out the edit, or the
649 if (! eb->called_open_root)
652 SVN_ERR(eb->wrapped_editor->open_root(eb->wrapped_edit_baton,
653 eb->base_revision, pool,
655 SVN_ERR(eb->wrapped_editor->close_directory(baton, pool));
660 if (eb->got_textdeltas)
661 SVN_ERR(svn_cmdline_printf(pool, "\n"));
662 if (eb->mergeinfo_tweaked)
663 SVN_ERR(svn_cmdline_printf(pool,
664 "NOTE: Adjusted Subversion mergeinfo in "
665 "this revision.\n"));
666 if (eb->mergeinfo_stripped)
667 SVN_ERR(svn_cmdline_printf(pool,
668 "NOTE: Dropped Subversion mergeinfo "
669 "from this revision.\n"));
670 if (eb->svnmerge_migrated)
671 SVN_ERR(svn_cmdline_printf(pool,
672 "NOTE: Migrated 'svnmerge-integrated' in "
673 "this revision.\n"));
674 if (eb->svnmerge_blocked)
675 SVN_ERR(svn_cmdline_printf(pool,
676 "NOTE: Saw 'svnmerge-blocked' in this "
677 "revision (but didn't migrate it).\n"));
680 return eb->wrapped_editor->close_edit(eb->wrapped_edit_baton, pool);
684 abort_edit(void *edit_baton,
687 edit_baton_t *eb = edit_baton;
688 return eb->wrapped_editor->abort_edit(eb->wrapped_edit_baton, pool);
692 /*** Editor factory function ***/
695 svnsync_get_sync_editor(const svn_delta_editor_t *wrapped_editor,
696 void *wrapped_edit_baton,
697 svn_revnum_t base_revision,
699 const char *source_prop_encoding,
701 const svn_delta_editor_t **editor,
703 int *normalized_node_props_counter,
706 svn_delta_editor_t *tree_editor = svn_delta_default_editor(pool);
707 edit_baton_t *eb = apr_pcalloc(pool, sizeof(*eb));
709 tree_editor->set_target_revision = set_target_revision;
710 tree_editor->open_root = open_root;
711 tree_editor->delete_entry = delete_entry;
712 tree_editor->add_directory = add_directory;
713 tree_editor->open_directory = open_directory;
714 tree_editor->change_dir_prop = change_dir_prop;
715 tree_editor->close_directory = close_directory;
716 tree_editor->absent_directory = absent_directory;
717 tree_editor->add_file = add_file;
718 tree_editor->open_file = open_file;
719 tree_editor->apply_textdelta = apply_textdelta;
720 tree_editor->change_file_prop = change_file_prop;
721 tree_editor->close_file = close_file;
722 tree_editor->absent_file = absent_file;
723 tree_editor->close_edit = close_edit;
724 tree_editor->abort_edit = abort_edit;
726 eb->wrapped_editor = wrapped_editor;
727 eb->wrapped_edit_baton = wrapped_edit_baton;
728 eb->base_revision = base_revision;
730 eb->source_prop_encoding = source_prop_encoding;
732 eb->normalized_node_props_counter = normalized_node_props_counter;
734 if (getenv("SVNSYNC_UNSUPPORTED_STRIP_MERGEINFO"))
736 eb->strip_mergeinfo = TRUE;
738 if (getenv("SVNSYNC_UNSUPPORTED_MIGRATE_SVNMERGE"))
740 /* Current we can't merge property values. That's only possible
741 if all the properties to be merged were always modified in
742 exactly the same revisions, or if we allow ourselves to
743 lookup the current state of properties in the sync
744 destination. So for now, migrating svnmerge.py data implies
745 stripping pre-existing svn:mergeinfo. */
746 /* ### FIXME: Do a real migration by consulting the mirror
747 ### repository's HEAD propvalues and merging svn:mergeinfo,
748 ### svnmerge-integrated, and svnmerge-blocked together. */
749 eb->migrate_svnmerge = TRUE;
750 eb->strip_mergeinfo = TRUE;
753 *editor = tree_editor;