2 * utf.c: UTF-8 conversion routines
4 * ====================================================================
5 * Licensed to the Apache Software Foundation (ASF) under one
6 * or more contributor license agreements. See the NOTICE file
7 * distributed with this work for additional information
8 * regarding copyright ownership. The ASF licenses this file
9 * to you under the Apache License, Version 2.0 (the
10 * "License"); you may not use this file except in compliance
11 * with the License. You may obtain a copy of the License at
13 * http://www.apache.org/licenses/LICENSE-2.0
15 * Unless required by applicable law or agreed to in writing,
16 * software distributed under the License is distributed on an
17 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
18 * KIND, either express or implied. See the License for the
19 * specific language governing permissions and limitations
21 * ====================================================================
30 #include <apr_strings.h>
32 #include <apr_xlate.h>
33 #include <apr_atomic.h>
36 #include "svn_string.h"
37 #include "svn_error.h"
38 #include "svn_pools.h"
39 #include "svn_ctype.h"
41 #include "svn_private_config.h"
42 #include "win32_xlate.h"
44 #include "private/svn_utf_private.h"
45 #include "private/svn_dep_compat.h"
46 #include "private/svn_string_private.h"
47 #include "private/svn_mutex.h"
51 /* Use these static strings to maximize performance on standard conversions.
52 * Any strings on other locations are still valid, however.
54 static const char *SVN_UTF_NTOU_XLATE_HANDLE = "svn-utf-ntou-xlate-handle";
55 static const char *SVN_UTF_UTON_XLATE_HANDLE = "svn-utf-uton-xlate-handle";
57 static const char *SVN_APR_UTF8_CHARSET = "UTF-8";
59 static svn_mutex__t *xlate_handle_mutex = NULL;
60 static svn_boolean_t assume_native_charset_is_utf8 = FALSE;
62 /* The xlate handle cache is a global hash table with linked lists of xlate
63 * handles. In multi-threaded environments, a thread "borrows" an xlate
64 * handle from the cache during a translation and puts it back afterwards.
65 * This avoids holding a global lock for all translations.
66 * If there is no handle for a particular key when needed, a new is
67 * handle is created and put in the cache after use.
68 * This means that there will be at most N handles open for a key, where N
69 * is the number of simultanous handles in use for that key. */
71 typedef struct xlate_handle_node_t {
73 /* FALSE if the handle is not valid, since its pool is being
76 /* The name of a char encoding or APR_LOCALE_CHARSET. */
77 const char *frompage, *topage;
78 struct xlate_handle_node_t *next;
79 } xlate_handle_node_t;
81 /* This maps const char * userdata_key strings to xlate_handle_node_t **
82 handles to the first entry in the linked list of xlate handles. We don't
83 store the pointer to the list head directly in the hash table, since we
84 remove/insert entries at the head in the list in the code below, and
85 we can't use apr_hash_set() in each character translation because that
86 function allocates memory in each call where the value is non-NULL.
87 Since these allocations take place in a global pool, this would be a
89 static apr_hash_t *xlate_handle_hash = NULL;
91 /* "1st level cache" to standard conversion maps. We may access these
92 * using atomic xchange ops, i.e. without further thread synchronization.
93 * If the respective item is NULL, fallback to hash lookup.
95 static void * volatile xlat_ntou_static_handle = NULL;
96 static void * volatile xlat_uton_static_handle = NULL;
98 /* Clean up the xlate handle cache. */
100 xlate_cleanup(void *arg)
102 /* We set the cache variables to NULL so that translation works in other
103 cleanup functions, even if it isn't cached then. */
104 xlate_handle_hash = NULL;
106 /* ensure no stale objects get accessed */
107 xlat_ntou_static_handle = NULL;
108 xlat_uton_static_handle = NULL;
113 /* Set the handle of ARG to NULL. */
115 xlate_handle_node_cleanup(void *arg)
117 xlate_handle_node_t *node = arg;
124 svn_utf_initialize2(svn_boolean_t assume_native_utf8,
127 if (!xlate_handle_hash)
129 /* We create our own subpool, which we protect with the mutex.
130 We can't use the pool passed to us by the caller, since we will
131 use it for xlate handle allocations, possibly in multiple threads,
132 and pool allocation is not thread-safe. */
133 apr_pool_t *subpool = svn_pool_create(pool);
135 svn_error_t *err = svn_mutex__init(&mutex, TRUE, subpool);
138 svn_error_clear(err);
142 xlate_handle_mutex = mutex;
143 xlate_handle_hash = apr_hash_make(subpool);
145 apr_pool_cleanup_register(subpool, NULL, xlate_cleanup,
146 apr_pool_cleanup_null);
149 if (!assume_native_charset_is_utf8)
150 assume_native_charset_is_utf8 = assume_native_utf8;
153 /* Return a unique string key based on TOPAGE and FROMPAGE. TOPAGE and
154 * FROMPAGE can be any valid arguments of the same name to
155 * apr_xlate_open(). Allocate the returned string in POOL. */
157 get_xlate_key(const char *topage,
158 const char *frompage,
161 /* In the cases of SVN_APR_LOCALE_CHARSET and SVN_APR_DEFAULT_CHARSET
162 * topage/frompage is really an int, not a valid string. So generate a
163 * unique key accordingly. */
164 if (frompage == SVN_APR_LOCALE_CHARSET)
165 frompage = "APR_LOCALE_CHARSET";
166 else if (frompage == SVN_APR_DEFAULT_CHARSET)
167 frompage = "APR_DEFAULT_CHARSET";
169 if (topage == SVN_APR_LOCALE_CHARSET)
170 topage = "APR_LOCALE_CHARSET";
171 else if (topage == SVN_APR_DEFAULT_CHARSET)
172 topage = "APR_DEFAULT_CHARSET";
174 return apr_pstrcat(pool, "svn-utf-", frompage, "to", topage,
175 "-xlate-handle", (char *)NULL);
178 /* Atomically replace the content in *MEM with NEW_VALUE and return
179 * the previous content of *MEM. If atomicy cannot be guaranteed,
180 * *MEM will not be modified and NEW_VALUE is simply returned to
183 static APR_INLINE void*
184 atomic_swap(void * volatile * mem, void *new_value)
187 #if APR_VERSION_AT_LEAST(1,3,0)
188 /* Cast is necessary because of APR bug:
189 https://issues.apache.org/bugzilla/show_bug.cgi?id=50731 */
190 return apr_atomic_xchgptr((volatile void **)mem, new_value);
192 /* old APRs don't support atomic swaps. Simply return the
193 * input to the caller for further proccessing. */
197 /* no threads - no sync. necessary */
198 void *old_value = (void*)*mem;
204 /* Set *RET to a newly created handle node for converting from FROMPAGE
205 to TOPAGE, If apr_xlate_open() returns APR_EINVAL or APR_ENOTIMPL, set
206 (*RET)->handle to NULL. If fail for any other reason, return the error.
207 Allocate *RET and its xlate handle in POOL. */
209 xlate_alloc_handle(xlate_handle_node_t **ret,
210 const char *topage, const char *frompage,
213 apr_status_t apr_err;
217 /* The error handling doesn't support the following cases, since we don't
218 use them currently. Catch this here. */
219 SVN_ERR_ASSERT(frompage != SVN_APR_DEFAULT_CHARSET
220 && topage != SVN_APR_DEFAULT_CHARSET
221 && (frompage != SVN_APR_LOCALE_CHARSET
222 || topage != SVN_APR_LOCALE_CHARSET));
224 /* Try to create a handle. */
226 apr_err = svn_subr__win32_xlate_open((win32_xlate_t **)&handle, topage,
228 name = "win32-xlate: ";
230 apr_err = apr_xlate_open(&handle, topage, frompage, pool);
234 if (APR_STATUS_IS_EINVAL(apr_err) || APR_STATUS_IS_ENOTIMPL(apr_err))
236 else if (apr_err != APR_SUCCESS)
239 char apr_strerr[512];
241 /* Can't use svn_error_wrap_apr here because it calls functions in
242 this file, leading to infinite recursion. */
243 if (frompage == SVN_APR_LOCALE_CHARSET)
244 errstr = apr_psprintf(pool,
245 _("Can't create a character converter from "
246 "native encoding to '%s'"), topage);
247 else if (topage == SVN_APR_LOCALE_CHARSET)
248 errstr = apr_psprintf(pool,
249 _("Can't create a character converter from "
250 "'%s' to native encoding"), frompage);
252 errstr = apr_psprintf(pool,
253 _("Can't create a character converter from "
254 "'%s' to '%s'"), frompage, topage);
256 /* Just put the error on the stack, since svn_error_create duplicates it
257 later. APR_STRERR will be in the local encoding, not in UTF-8, though.
259 svn_strerror(apr_err, apr_strerr, sizeof(apr_strerr));
260 return svn_error_createf(SVN_ERR_PLUGIN_LOAD_FAILURE,
261 svn_error_create(apr_err, NULL, apr_strerr),
262 "%s%s", name, errstr);
265 /* Allocate and initialize the node. */
266 *ret = apr_palloc(pool, sizeof(xlate_handle_node_t));
267 (*ret)->handle = handle;
268 (*ret)->valid = TRUE;
269 (*ret)->frompage = ((frompage != SVN_APR_LOCALE_CHARSET)
270 ? apr_pstrdup(pool, frompage) : frompage);
271 (*ret)->topage = ((topage != SVN_APR_LOCALE_CHARSET)
272 ? apr_pstrdup(pool, topage) : topage);
275 /* If we are called from inside a pool cleanup handler, the just created
276 xlate handle will be closed when that handler returns by a newly
277 registered cleanup handler, however, the handle is still cached by us.
278 To prevent this, we register a cleanup handler that will reset the valid
279 flag of our node, so we don't use an invalid handle. */
281 apr_pool_cleanup_register(pool, *ret, xlate_handle_node_cleanup,
282 apr_pool_cleanup_null);
287 /* Extend xlate_alloc_handle by using USERDATA_KEY as a key in our
288 global hash map, if available.
290 Allocate *RET and its xlate handle in POOL if svn_utf_initialize()
291 hasn't been called or USERDATA_KEY is NULL. Else, allocate them
292 in the pool of xlate_handle_hash.
294 Note: this function is not thread-safe. Call get_xlate_handle_node
297 get_xlate_handle_node_internal(xlate_handle_node_t **ret,
298 const char *topage, const char *frompage,
299 const char *userdata_key, apr_pool_t *pool)
301 /* If we already have a handle, just return it. */
302 if (userdata_key && xlate_handle_hash)
304 xlate_handle_node_t *old_node = NULL;
306 /* 2nd level: hash lookup */
307 xlate_handle_node_t **old_node_p = svn_hash_gets(xlate_handle_hash,
310 old_node = *old_node_p;
313 /* Ensure that the handle is still valid. */
316 /* Remove from the list. */
317 *old_node_p = old_node->next;
318 old_node->next = NULL;
325 /* Note that we still have the mutex locked (if it is initialized), so we
326 can use the global pool for creating the new xlate handle. */
328 /* Use the correct pool for creating the handle. */
329 pool = apr_hash_pool_get(xlate_handle_hash);
331 return xlate_alloc_handle(ret, topage, frompage, pool);
334 /* Set *RET to a handle node for converting from FROMPAGE to TOPAGE,
335 creating the handle node if it doesn't exist in USERDATA_KEY.
336 If a node is not cached and apr_xlate_open() returns APR_EINVAL or
337 APR_ENOTIMPL, set (*RET)->handle to NULL. If fail for any other
338 reason, return the error.
340 Allocate *RET and its xlate handle in POOL if svn_utf_initialize()
341 hasn't been called or USERDATA_KEY is NULL. Else, allocate them
342 in the pool of xlate_handle_hash. */
344 get_xlate_handle_node(xlate_handle_node_t **ret,
345 const char *topage, const char *frompage,
346 const char *userdata_key, apr_pool_t *pool)
348 xlate_handle_node_t *old_node = NULL;
350 /* If we already have a handle, just return it. */
353 if (xlate_handle_hash)
355 /* 1st level: global, static items */
356 if (userdata_key == SVN_UTF_NTOU_XLATE_HANDLE)
357 old_node = atomic_swap(&xlat_ntou_static_handle, NULL);
358 else if (userdata_key == SVN_UTF_UTON_XLATE_HANDLE)
359 old_node = atomic_swap(&xlat_uton_static_handle, NULL);
361 if (old_node && old_node->valid)
370 /* We fall back on a per-pool cache instead. */
371 apr_pool_userdata_get(&p, userdata_key, pool);
373 /* Ensure that the handle is still valid. */
374 if (old_node && old_node->valid)
380 return xlate_alloc_handle(ret, topage, frompage, pool);
384 SVN_MUTEX__WITH_LOCK(xlate_handle_mutex,
385 get_xlate_handle_node_internal(ret,
394 /* Put back NODE into the xlate handle cache for use by other calls.
396 Note: this function is not thread-safe. Call put_xlate_handle_node
399 put_xlate_handle_node_internal(xlate_handle_node_t *node,
400 const char *userdata_key)
402 xlate_handle_node_t **node_p = svn_hash_gets(xlate_handle_hash, userdata_key);
405 userdata_key = apr_pstrdup(apr_hash_pool_get(xlate_handle_hash),
407 node_p = apr_palloc(apr_hash_pool_get(xlate_handle_hash),
410 svn_hash_sets(xlate_handle_hash, userdata_key, node_p);
412 node->next = *node_p;
418 /* Put back NODE into the xlate handle cache for use by other calls.
419 If there is no global cache, store the handle in POOL.
420 Ignore errors related to locking/unlocking the mutex. */
422 put_xlate_handle_node(xlate_handle_node_t *node,
423 const char *userdata_key,
426 assert(node->next == NULL);
430 /* push previous global node to the hash */
431 if (xlate_handle_hash)
433 /* 1st level: global, static items */
434 if (userdata_key == SVN_UTF_NTOU_XLATE_HANDLE)
435 node = atomic_swap(&xlat_ntou_static_handle, node);
436 else if (userdata_key == SVN_UTF_UTON_XLATE_HANDLE)
437 node = atomic_swap(&xlat_uton_static_handle, node);
441 SVN_MUTEX__WITH_LOCK(xlate_handle_mutex,
442 put_xlate_handle_node_internal(node,
447 /* Store it in the per-pool cache. */
448 apr_pool_userdata_set(node, userdata_key, apr_pool_cleanup_null, pool);
454 /* Return the apr_xlate handle for converting native characters to UTF-8. */
456 get_ntou_xlate_handle_node(xlate_handle_node_t **ret, apr_pool_t *pool)
458 return get_xlate_handle_node(ret, SVN_APR_UTF8_CHARSET,
459 assume_native_charset_is_utf8
460 ? SVN_APR_UTF8_CHARSET
461 : SVN_APR_LOCALE_CHARSET,
462 SVN_UTF_NTOU_XLATE_HANDLE, pool);
466 /* Return the apr_xlate handle for converting UTF-8 to native characters.
467 Create one if it doesn't exist. If unable to find a handle, or
468 unable to create one because apr_xlate_open returned APR_EINVAL, then
469 set *RET to null and return SVN_NO_ERROR; if fail for some other
470 reason, return error. */
472 get_uton_xlate_handle_node(xlate_handle_node_t **ret, apr_pool_t *pool)
474 return get_xlate_handle_node(ret,
475 assume_native_charset_is_utf8
476 ? SVN_APR_UTF8_CHARSET
477 : SVN_APR_LOCALE_CHARSET,
478 SVN_APR_UTF8_CHARSET,
479 SVN_UTF_UTON_XLATE_HANDLE, pool);
483 /* Copy LEN bytes of SRC, converting non-ASCII and zero bytes to ?\nnn
484 sequences, allocating the result in POOL. */
486 fuzzy_escape(const char *src, apr_size_t len, apr_pool_t *pool)
488 const char *src_orig = src, *src_end = src + len;
489 apr_size_t new_len = 0;
491 const char *new_orig;
493 /* First count how big a dest string we'll need. */
494 while (src < src_end)
496 if (! svn_ctype_isascii(*src) || *src == '\0')
497 new_len += 5; /* 5 slots, for "?\XXX" */
499 new_len += 1; /* one slot for the 7-bit char */
504 /* Allocate that amount, plus one slot for '\0' character. */
505 new = apr_palloc(pool, new_len + 1);
509 /* And fill it up. */
510 while (src_orig < src_end)
512 if (! svn_ctype_isascii(*src_orig) || src_orig == '\0')
514 /* This is the same format as svn_xml_fuzzy_escape uses, but that
515 function escapes different characters. Please keep in sync!
516 ### If we add another fuzzy escape somewhere, we should abstract
517 ### this out to a common function. */
518 apr_snprintf(new, 6, "?\\%03u", (unsigned char) *src_orig);
535 /* Convert SRC_LENGTH bytes of SRC_DATA in NODE->handle, store the result
536 in *DEST, which is allocated in POOL. */
538 convert_to_stringbuf(xlate_handle_node_t *node,
539 const char *src_data,
540 apr_size_t src_length,
541 svn_stringbuf_t **dest,
545 apr_status_t apr_err;
547 apr_err = svn_subr__win32_xlate_to_stringbuf((win32_xlate_t *) node->handle,
548 src_data, src_length,
551 apr_size_t buflen = src_length * 2;
552 apr_status_t apr_err;
553 apr_size_t srclen = src_length;
554 apr_size_t destlen = buflen;
556 /* Initialize *DEST to an empty stringbuf.
557 A 1:2 ratio of input bytes to output bytes (as assigned above)
558 should be enough for most translations, and if it turns out not
559 to be enough, we'll grow the buffer again, sizing it based on a
560 1:3 ratio of the remainder of the string. */
561 *dest = svn_stringbuf_create_ensure(buflen + 1, pool);
563 /* Not only does it not make sense to convert an empty string, but
564 apr-iconv is quite unreasonable about not allowing that. */
570 /* Set up state variables for xlate. */
571 destlen = buflen - (*dest)->len;
573 /* Attempt the conversion. */
574 apr_err = apr_xlate_conv_buffer(node->handle,
575 src_data + (src_length - srclen),
577 (*dest)->data + (*dest)->len,
580 /* Now, update the *DEST->len to track the amount of output data
581 churned out so far from this loop. */
582 (*dest)->len += ((buflen - (*dest)->len) - destlen);
583 buflen += srclen * 3; /* 3 is middle ground, 2 wasn't enough
584 for all characters in the buffer, 4 is
585 maximum character size (currently) */
588 } while (apr_err == APR_SUCCESS && srclen != 0);
591 /* If we exited the loop with an error, return the error. */
597 /* Can't use svn_error_wrap_apr here because it calls functions in
598 this file, leading to infinite recursion. */
599 if (node->frompage == SVN_APR_LOCALE_CHARSET)
600 errstr = apr_psprintf
601 (pool, _("Can't convert string from native encoding to '%s':"),
603 else if (node->topage == SVN_APR_LOCALE_CHARSET)
604 errstr = apr_psprintf
605 (pool, _("Can't convert string from '%s' to native encoding:"),
608 errstr = apr_psprintf
609 (pool, _("Can't convert string from '%s' to '%s':"),
610 node->frompage, node->topage);
612 err = svn_error_create(apr_err, NULL, fuzzy_escape(src_data,
614 return svn_error_create(apr_err, err, errstr);
616 /* Else, exited due to success. Trim the result buffer down to the
618 (*dest)->data[(*dest)->len] = '\0';
624 /* Return APR_EINVAL if the first LEN bytes of DATA contain anything
625 other than seven-bit, non-control (except for whitespace) ASCII
626 characters, finding the error pool from POOL. Otherwise, return
629 check_non_ascii(const char *data, apr_size_t len, apr_pool_t *pool)
631 const char *data_start = data;
633 for (; len > 0; --len, data++)
635 if ((! svn_ctype_isascii(*data))
636 || ((! svn_ctype_isspace(*data))
637 && svn_ctype_iscntrl(*data)))
639 /* Show the printable part of the data, followed by the
640 decimal code of the questionable character. Because if a
641 user ever gets this error, she's going to have to spend
642 time tracking down the non-ASCII data, so we want to help
643 as much as possible. And yes, we just call the unsafe
644 data "non-ASCII", even though the actual constraint is
645 somewhat more complex than that. */
647 if (data - data_start)
649 const char *error_data
650 = apr_pstrndup(pool, data_start, (data - data_start));
652 return svn_error_createf
654 _("Safe data '%s' was followed by non-ASCII byte %d: "
655 "unable to convert to/from UTF-8"),
656 error_data, *((const unsigned char *) data));
660 return svn_error_createf
662 _("Non-ASCII character (code %d) detected, "
663 "and unable to convert to/from UTF-8"),
664 *((const unsigned char *) data));
672 /* Construct an error with code APR_EINVAL and with a suitable message
673 * to describe the invalid UTF-8 sequence DATA of length LEN (which
674 * may have embedded NULLs). We can't simply print the data, almost
675 * by definition we don't really know how it is encoded.
678 invalid_utf8(const char *data, apr_size_t len, apr_pool_t *pool)
680 const char *last = svn_utf__last_valid(data, len);
681 const char *valid_txt = "", *invalid_txt = "";
683 size_t valid, invalid;
685 /* We will display at most 24 valid octets (this may split a leading
686 multi-byte character) as that should fit on one 80 character line. */
690 for (i = 0; i < valid; ++i)
691 valid_txt = apr_pstrcat(pool, valid_txt,
692 apr_psprintf(pool, " %02x",
693 (unsigned char)last[i-valid]),
696 /* 4 invalid octets will guarantee that the faulty octet is displayed */
697 invalid = data + len - last;
700 for (i = 0; i < invalid; ++i)
701 invalid_txt = apr_pstrcat(pool, invalid_txt,
702 apr_psprintf(pool, " %02x",
703 (unsigned char)last[i]),
706 return svn_error_createf(APR_EINVAL, NULL,
707 _("Valid UTF-8 data\n(hex:%s)\n"
708 "followed by invalid UTF-8 sequence\n(hex:%s)"),
709 valid_txt, invalid_txt);
712 /* Verify that the sequence DATA of length LEN is valid UTF-8.
713 If it is not, return an error with code APR_EINVAL. */
715 check_utf8(const char *data, apr_size_t len, apr_pool_t *pool)
717 if (! svn_utf__is_valid(data, len))
718 return invalid_utf8(data, len, pool);
722 /* Verify that the NULL terminated sequence DATA is valid UTF-8.
723 If it is not, return an error with code APR_EINVAL. */
725 check_cstring_utf8(const char *data, apr_pool_t *pool)
728 if (! svn_utf__cstring_is_valid(data))
729 return invalid_utf8(data, strlen(data), pool);
735 svn_utf_stringbuf_to_utf8(svn_stringbuf_t **dest,
736 const svn_stringbuf_t *src,
739 xlate_handle_node_t *node;
742 SVN_ERR(get_ntou_xlate_handle_node(&node, pool));
746 err = convert_to_stringbuf(node, src->data, src->len, dest, pool);
748 err = check_utf8((*dest)->data, (*dest)->len, pool);
752 err = check_non_ascii(src->data, src->len, pool);
754 *dest = svn_stringbuf_dup(src, pool);
757 return svn_error_compose_create(err,
758 put_xlate_handle_node
760 SVN_UTF_NTOU_XLATE_HANDLE,
766 svn_utf_string_to_utf8(const svn_string_t **dest,
767 const svn_string_t *src,
770 svn_stringbuf_t *destbuf;
771 xlate_handle_node_t *node;
774 SVN_ERR(get_ntou_xlate_handle_node(&node, pool));
778 err = convert_to_stringbuf(node, src->data, src->len, &destbuf, pool);
780 err = check_utf8(destbuf->data, destbuf->len, pool);
782 *dest = svn_stringbuf__morph_into_string(destbuf);
786 err = check_non_ascii(src->data, src->len, pool);
788 *dest = svn_string_dup(src, pool);
791 return svn_error_compose_create(err,
792 put_xlate_handle_node
794 SVN_UTF_NTOU_XLATE_HANDLE,
799 /* Common implementation for svn_utf_cstring_to_utf8,
800 svn_utf_cstring_to_utf8_ex, svn_utf_cstring_from_utf8 and
801 svn_utf_cstring_from_utf8_ex. Convert SRC to DEST using NODE->handle as
802 the translator and allocating from POOL. */
804 convert_cstring(const char **dest,
806 xlate_handle_node_t *node,
811 svn_stringbuf_t *destbuf;
812 SVN_ERR(convert_to_stringbuf(node, src, strlen(src),
814 *dest = destbuf->data;
818 apr_size_t len = strlen(src);
819 SVN_ERR(check_non_ascii(src, len, pool));
820 *dest = apr_pstrmemdup(pool, src, len);
827 svn_utf_cstring_to_utf8(const char **dest,
831 xlate_handle_node_t *node;
834 SVN_ERR(get_ntou_xlate_handle_node(&node, pool));
835 err = convert_cstring(dest, src, node, pool);
836 SVN_ERR(svn_error_compose_create(err,
837 put_xlate_handle_node
839 SVN_UTF_NTOU_XLATE_HANDLE,
841 return check_cstring_utf8(*dest, pool);
846 svn_utf_cstring_to_utf8_ex2(const char **dest,
848 const char *frompage,
851 xlate_handle_node_t *node;
853 const char *convset_key = get_xlate_key(SVN_APR_UTF8_CHARSET, frompage,
856 SVN_ERR(get_xlate_handle_node(&node, SVN_APR_UTF8_CHARSET, frompage,
858 err = convert_cstring(dest, src, node, pool);
859 SVN_ERR(svn_error_compose_create(err,
860 put_xlate_handle_node
862 SVN_UTF_NTOU_XLATE_HANDLE,
865 return check_cstring_utf8(*dest, pool);
870 svn_utf_cstring_to_utf8_ex(const char **dest,
872 const char *frompage,
873 const char *convset_key,
876 return svn_utf_cstring_to_utf8_ex2(dest, src, frompage, pool);
881 svn_utf_stringbuf_from_utf8(svn_stringbuf_t **dest,
882 const svn_stringbuf_t *src,
885 xlate_handle_node_t *node;
888 SVN_ERR(get_uton_xlate_handle_node(&node, pool));
892 err = check_utf8(src->data, src->len, pool);
894 err = convert_to_stringbuf(node, src->data, src->len, dest, pool);
898 err = check_non_ascii(src->data, src->len, pool);
900 *dest = svn_stringbuf_dup(src, pool);
903 err = svn_error_compose_create(
905 put_xlate_handle_node(node, SVN_UTF_UTON_XLATE_HANDLE, pool));
912 svn_utf_string_from_utf8(const svn_string_t **dest,
913 const svn_string_t *src,
916 svn_stringbuf_t *dbuf;
917 xlate_handle_node_t *node;
920 SVN_ERR(get_uton_xlate_handle_node(&node, pool));
924 err = check_utf8(src->data, src->len, pool);
926 err = convert_to_stringbuf(node, src->data, src->len,
929 *dest = svn_stringbuf__morph_into_string(dbuf);
933 err = check_non_ascii(src->data, src->len, pool);
935 *dest = svn_string_dup(src, pool);
938 err = svn_error_compose_create(
940 put_xlate_handle_node(node, SVN_UTF_UTON_XLATE_HANDLE, pool));
947 svn_utf_cstring_from_utf8(const char **dest,
951 xlate_handle_node_t *node;
954 SVN_ERR(check_cstring_utf8(src, pool));
956 SVN_ERR(get_uton_xlate_handle_node(&node, pool));
957 err = convert_cstring(dest, src, node, pool);
958 err = svn_error_compose_create(
960 put_xlate_handle_node(node, SVN_UTF_UTON_XLATE_HANDLE, pool));
967 svn_utf_cstring_from_utf8_ex2(const char **dest,
972 xlate_handle_node_t *node;
974 const char *convset_key = get_xlate_key(topage, SVN_APR_UTF8_CHARSET,
977 SVN_ERR(check_cstring_utf8(src, pool));
979 SVN_ERR(get_xlate_handle_node(&node, topage, SVN_APR_UTF8_CHARSET,
981 err = convert_cstring(dest, src, node, pool);
982 err = svn_error_compose_create(
984 put_xlate_handle_node(node, convset_key, pool));
991 svn_utf_cstring_from_utf8_ex(const char **dest,
994 const char *convset_key,
997 return svn_utf_cstring_from_utf8_ex2(dest, src, topage, pool);
1002 svn_utf__cstring_from_utf8_fuzzy(const char *src,
1004 svn_error_t *(*convert_from_utf8)
1005 (const char **, const char *, apr_pool_t *))
1007 const char *escaped, *converted;
1010 escaped = fuzzy_escape(src, strlen(src), pool);
1012 /* Okay, now we have a *new* UTF-8 string, one that's guaranteed to
1013 contain only 7-bit bytes :-). Recode to native... */
1014 err = convert_from_utf8(((const char **) &converted), escaped, pool);
1018 svn_error_clear(err);
1024 /* ### Check the client locale, maybe we can avoid that second
1025 * conversion! See Ulrich Drepper's patch at
1026 * http://subversion.tigris.org/issues/show_bug.cgi?id=807.
1032 svn_utf_cstring_from_utf8_fuzzy(const char *src,
1035 return svn_utf__cstring_from_utf8_fuzzy(src, pool,
1036 svn_utf_cstring_from_utf8);
1041 svn_utf_cstring_from_utf8_stringbuf(const char **dest,
1042 const svn_stringbuf_t *src,
1045 svn_stringbuf_t *destbuf;
1047 SVN_ERR(svn_utf_stringbuf_from_utf8(&destbuf, src, pool));
1048 *dest = destbuf->data;
1050 return SVN_NO_ERROR;
1055 svn_utf_cstring_from_utf8_string(const char **dest,
1056 const svn_string_t *src,
1059 svn_stringbuf_t *dbuf;
1060 xlate_handle_node_t *node;
1063 SVN_ERR(get_uton_xlate_handle_node(&node, pool));
1067 err = check_utf8(src->data, src->len, pool);
1069 err = convert_to_stringbuf(node, src->data, src->len,
1076 err = check_non_ascii(src->data, src->len, pool);
1078 *dest = apr_pstrmemdup(pool, src->data, src->len);
1081 err = svn_error_compose_create(
1083 put_xlate_handle_node(node, SVN_UTF_UTON_XLATE_HANDLE, pool));