2 * utf.c: UTF-8 conversion routines
4 * ====================================================================
5 * Licensed to the Apache Software Foundation (ASF) under one
6 * or more contributor license agreements. See the NOTICE file
7 * distributed with this work for additional information
8 * regarding copyright ownership. The ASF licenses this file
9 * to you under the Apache License, Version 2.0 (the
10 * "License"); you may not use this file except in compliance
11 * with the License. You may obtain a copy of the License at
13 * http://www.apache.org/licenses/LICENSE-2.0
15 * Unless required by applicable law or agreed to in writing,
16 * software distributed under the License is distributed on an
17 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
18 * KIND, either express or implied. See the License for the
19 * specific language governing permissions and limitations
21 * ====================================================================
30 #include <apr_strings.h>
32 #include <apr_xlate.h>
33 #include <apr_atomic.h>
36 #include "svn_string.h"
37 #include "svn_error.h"
38 #include "svn_pools.h"
39 #include "svn_ctype.h"
41 #include "svn_private_config.h"
42 #include "win32_xlate.h"
44 #include "private/svn_utf_private.h"
45 #include "private/svn_dep_compat.h"
46 #include "private/svn_string_private.h"
47 #include "private/svn_mutex.h"
51 /* Use these static strings to maximize performance on standard conversions.
52 * Any strings on other locations are still valid, however.
54 static const char *SVN_UTF_NTOU_XLATE_HANDLE = "svn-utf-ntou-xlate-handle";
55 static const char *SVN_UTF_UTON_XLATE_HANDLE = "svn-utf-uton-xlate-handle";
57 static const char *SVN_APR_UTF8_CHARSET = "UTF-8";
59 static svn_mutex__t *xlate_handle_mutex = NULL;
60 static svn_boolean_t assume_native_charset_is_utf8 = FALSE;
63 typedef svn_subr__win32_xlate_t xlate_handle_t;
65 typedef apr_xlate_t xlate_handle_t;
68 /* The xlate handle cache is a global hash table with linked lists of xlate
69 * handles. In multi-threaded environments, a thread "borrows" an xlate
70 * handle from the cache during a translation and puts it back afterwards.
71 * This avoids holding a global lock for all translations.
72 * If there is no handle for a particular key when needed, a new is
73 * handle is created and put in the cache after use.
74 * This means that there will be at most N handles open for a key, where N
75 * is the number of simultanous handles in use for that key. */
77 typedef struct xlate_handle_node_t {
78 xlate_handle_t *handle;
79 /* FALSE if the handle is not valid, since its pool is being
82 /* The name of a char encoding or APR_LOCALE_CHARSET. */
83 const char *frompage, *topage;
84 struct xlate_handle_node_t *next;
85 } xlate_handle_node_t;
87 /* This maps const char * userdata_key strings to xlate_handle_node_t **
88 handles to the first entry in the linked list of xlate handles. We don't
89 store the pointer to the list head directly in the hash table, since we
90 remove/insert entries at the head in the list in the code below, and
91 we can't use apr_hash_set() in each character translation because that
92 function allocates memory in each call where the value is non-NULL.
93 Since these allocations take place in a global pool, this would be a
95 static apr_hash_t *xlate_handle_hash = NULL;
97 /* "1st level cache" to standard conversion maps. We may access these
98 * using atomic xchange ops, i.e. without further thread synchronization.
99 * If the respective item is NULL, fallback to hash lookup.
101 static void * volatile xlat_ntou_static_handle = NULL;
102 static void * volatile xlat_uton_static_handle = NULL;
104 /* Clean up the xlate handle cache. */
106 xlate_cleanup(void *arg)
108 /* We set the cache variables to NULL so that translation works in other
109 cleanup functions, even if it isn't cached then. */
110 xlate_handle_hash = NULL;
112 /* ensure no stale objects get accessed */
113 xlat_ntou_static_handle = NULL;
114 xlat_uton_static_handle = NULL;
119 /* Set the handle of ARG to NULL. */
121 xlate_handle_node_cleanup(void *arg)
123 xlate_handle_node_t *node = arg;
130 svn_utf_initialize2(svn_boolean_t assume_native_utf8,
133 if (!xlate_handle_hash)
135 /* We create our own subpool, which we protect with the mutex.
136 We can't use the pool passed to us by the caller, since we will
137 use it for xlate handle allocations, possibly in multiple threads,
138 and pool allocation is not thread-safe. */
139 apr_pool_t *subpool = svn_pool_create(pool);
141 svn_error_t *err = svn_mutex__init(&mutex, TRUE, subpool);
144 svn_error_clear(err);
148 xlate_handle_mutex = mutex;
149 xlate_handle_hash = apr_hash_make(subpool);
151 apr_pool_cleanup_register(subpool, NULL, xlate_cleanup,
152 apr_pool_cleanup_null);
155 if (!assume_native_charset_is_utf8)
156 assume_native_charset_is_utf8 = assume_native_utf8;
159 /* Return a unique string key based on TOPAGE and FROMPAGE. TOPAGE and
160 * FROMPAGE can be any valid arguments of the same name to
161 * apr_xlate_open(). Allocate the returned string in POOL. */
163 get_xlate_key(const char *topage,
164 const char *frompage,
167 /* In the cases of SVN_APR_LOCALE_CHARSET and SVN_APR_DEFAULT_CHARSET
168 * topage/frompage is really an int, not a valid string. So generate a
169 * unique key accordingly. */
170 if (frompage == SVN_APR_LOCALE_CHARSET)
171 frompage = "APR_LOCALE_CHARSET";
172 else if (frompage == SVN_APR_DEFAULT_CHARSET)
173 frompage = "APR_DEFAULT_CHARSET";
175 if (topage == SVN_APR_LOCALE_CHARSET)
176 topage = "APR_LOCALE_CHARSET";
177 else if (topage == SVN_APR_DEFAULT_CHARSET)
178 topage = "APR_DEFAULT_CHARSET";
180 return apr_pstrcat(pool, "svn-utf-", frompage, "to", topage,
181 "-xlate-handle", SVN_VA_NULL);
184 /* Atomically replace the content in *MEM with NEW_VALUE and return
185 * the previous content of *MEM. If atomicy cannot be guaranteed,
186 * *MEM will not be modified and NEW_VALUE is simply returned to
189 static APR_INLINE void*
190 atomic_swap(void * volatile * mem, void *new_value)
193 /* Cast is necessary because of APR bug:
194 https://issues.apache.org/bugzilla/show_bug.cgi?id=50731 */
195 return apr_atomic_xchgptr((volatile void **)mem, new_value);
197 /* no threads - no sync. necessary */
198 void *old_value = (void*)*mem;
204 /* Set *RET to a newly created handle node for converting from FROMPAGE
205 to TOPAGE, If apr_xlate_open() returns APR_EINVAL or APR_ENOTIMPL, set
206 (*RET)->handle to NULL. If fail for any other reason, return the error.
207 Allocate *RET and its xlate handle in POOL. */
209 xlate_alloc_handle(xlate_handle_node_t **ret,
210 const char *topage, const char *frompage,
213 apr_status_t apr_err;
214 xlate_handle_t *handle;
217 /* The error handling doesn't support the following cases, since we don't
218 use them currently. Catch this here. */
219 SVN_ERR_ASSERT(frompage != SVN_APR_DEFAULT_CHARSET
220 && topage != SVN_APR_DEFAULT_CHARSET
221 && (frompage != SVN_APR_LOCALE_CHARSET
222 || topage != SVN_APR_LOCALE_CHARSET));
224 /* Try to create a handle. */
226 apr_err = svn_subr__win32_xlate_open(&handle, topage,
228 name = "win32-xlate: ";
230 apr_err = apr_xlate_open(&handle, topage, frompage, pool);
234 if (APR_STATUS_IS_EINVAL(apr_err) || APR_STATUS_IS_ENOTIMPL(apr_err))
236 else if (apr_err != APR_SUCCESS)
239 char apr_strerr[512];
241 /* Can't use svn_error_wrap_apr here because it calls functions in
242 this file, leading to infinite recursion. */
243 if (frompage == SVN_APR_LOCALE_CHARSET)
244 errstr = apr_psprintf(pool,
245 _("Can't create a character converter from "
246 "native encoding to '%s'"), topage);
247 else if (topage == SVN_APR_LOCALE_CHARSET)
248 errstr = apr_psprintf(pool,
249 _("Can't create a character converter from "
250 "'%s' to native encoding"), frompage);
252 errstr = apr_psprintf(pool,
253 _("Can't create a character converter from "
254 "'%s' to '%s'"), frompage, topage);
256 /* Just put the error on the stack, since svn_error_create duplicates it
257 later. APR_STRERR will be in the local encoding, not in UTF-8, though.
259 svn_strerror(apr_err, apr_strerr, sizeof(apr_strerr));
260 return svn_error_createf(SVN_ERR_PLUGIN_LOAD_FAILURE,
261 svn_error_create(apr_err, NULL, apr_strerr),
262 "%s%s", name, errstr);
265 /* Allocate and initialize the node. */
266 *ret = apr_palloc(pool, sizeof(xlate_handle_node_t));
267 (*ret)->handle = handle;
268 (*ret)->valid = TRUE;
269 (*ret)->frompage = ((frompage != SVN_APR_LOCALE_CHARSET)
270 ? apr_pstrdup(pool, frompage) : frompage);
271 (*ret)->topage = ((topage != SVN_APR_LOCALE_CHARSET)
272 ? apr_pstrdup(pool, topage) : topage);
275 /* If we are called from inside a pool cleanup handler, the just created
276 xlate handle will be closed when that handler returns by a newly
277 registered cleanup handler, however, the handle is still cached by us.
278 To prevent this, we register a cleanup handler that will reset the valid
279 flag of our node, so we don't use an invalid handle. */
281 apr_pool_cleanup_register(pool, *ret, xlate_handle_node_cleanup,
282 apr_pool_cleanup_null);
287 /* Extend xlate_alloc_handle by using USERDATA_KEY as a key in our
288 global hash map, if available.
290 Allocate *RET and its xlate handle in POOL if svn_utf_initialize()
291 hasn't been called or USERDATA_KEY is NULL. Else, allocate them
292 in the pool of xlate_handle_hash.
294 Note: this function is not thread-safe. Call get_xlate_handle_node
297 get_xlate_handle_node_internal(xlate_handle_node_t **ret,
298 const char *topage, const char *frompage,
299 const char *userdata_key, apr_pool_t *pool)
301 /* If we already have a handle, just return it. */
302 if (userdata_key && xlate_handle_hash)
304 xlate_handle_node_t *old_node = NULL;
306 /* 2nd level: hash lookup */
307 xlate_handle_node_t **old_node_p = svn_hash_gets(xlate_handle_hash,
310 old_node = *old_node_p;
313 /* Ensure that the handle is still valid. */
316 /* Remove from the list. */
317 *old_node_p = old_node->next;
318 old_node->next = NULL;
325 /* Note that we still have the mutex locked (if it is initialized), so we
326 can use the global pool for creating the new xlate handle. */
328 /* Use the correct pool for creating the handle. */
329 pool = apr_hash_pool_get(xlate_handle_hash);
331 return xlate_alloc_handle(ret, topage, frompage, pool);
334 /* Set *RET to a handle node for converting from FROMPAGE to TOPAGE,
335 creating the handle node if it doesn't exist in USERDATA_KEY.
336 If a node is not cached and apr_xlate_open() returns APR_EINVAL or
337 APR_ENOTIMPL, set (*RET)->handle to NULL. If fail for any other
338 reason, return the error.
340 Allocate *RET and its xlate handle in POOL if svn_utf_initialize()
341 hasn't been called or USERDATA_KEY is NULL. Else, allocate them
342 in the pool of xlate_handle_hash. */
344 get_xlate_handle_node(xlate_handle_node_t **ret,
345 const char *topage, const char *frompage,
346 const char *userdata_key, apr_pool_t *pool)
348 xlate_handle_node_t *old_node = NULL;
350 /* If we already have a handle, just return it. */
353 if (xlate_handle_hash)
355 /* 1st level: global, static items */
356 if (userdata_key == SVN_UTF_NTOU_XLATE_HANDLE)
357 old_node = atomic_swap(&xlat_ntou_static_handle, NULL);
358 else if (userdata_key == SVN_UTF_UTON_XLATE_HANDLE)
359 old_node = atomic_swap(&xlat_uton_static_handle, NULL);
361 if (old_node && old_node->valid)
370 /* We fall back on a per-pool cache instead. */
371 apr_pool_userdata_get(&p, userdata_key, pool);
373 /* Ensure that the handle is still valid. */
374 if (old_node && old_node->valid)
380 return xlate_alloc_handle(ret, topage, frompage, pool);
384 SVN_MUTEX__WITH_LOCK(xlate_handle_mutex,
385 get_xlate_handle_node_internal(ret,
394 /* Put back NODE into the xlate handle cache for use by other calls.
396 Note: this function is not thread-safe. Call put_xlate_handle_node
399 put_xlate_handle_node_internal(xlate_handle_node_t *node,
400 const char *userdata_key)
402 xlate_handle_node_t **node_p = svn_hash_gets(xlate_handle_hash, userdata_key);
405 userdata_key = apr_pstrdup(apr_hash_pool_get(xlate_handle_hash),
407 node_p = apr_palloc(apr_hash_pool_get(xlate_handle_hash),
410 svn_hash_sets(xlate_handle_hash, userdata_key, node_p);
412 node->next = *node_p;
418 /* Put back NODE into the xlate handle cache for use by other calls.
419 If there is no global cache, store the handle in POOL.
420 Ignore errors related to locking/unlocking the mutex. */
422 put_xlate_handle_node(xlate_handle_node_t *node,
423 const char *userdata_key,
426 assert(node->next == NULL);
430 /* push previous global node to the hash */
431 if (xlate_handle_hash)
433 /* 1st level: global, static items */
434 if (userdata_key == SVN_UTF_NTOU_XLATE_HANDLE)
435 node = atomic_swap(&xlat_ntou_static_handle, node);
436 else if (userdata_key == SVN_UTF_UTON_XLATE_HANDLE)
437 node = atomic_swap(&xlat_uton_static_handle, node);
441 SVN_MUTEX__WITH_LOCK(xlate_handle_mutex,
442 put_xlate_handle_node_internal(node,
447 /* Store it in the per-pool cache. */
448 apr_pool_userdata_set(node, userdata_key, apr_pool_cleanup_null, pool);
454 /* Return the apr_xlate handle for converting native characters to UTF-8. */
456 get_ntou_xlate_handle_node(xlate_handle_node_t **ret, apr_pool_t *pool)
458 return get_xlate_handle_node(ret, SVN_APR_UTF8_CHARSET,
459 assume_native_charset_is_utf8
460 ? SVN_APR_UTF8_CHARSET
461 : SVN_APR_LOCALE_CHARSET,
462 SVN_UTF_NTOU_XLATE_HANDLE, pool);
466 /* Return the apr_xlate handle for converting UTF-8 to native characters.
467 Create one if it doesn't exist. If unable to find a handle, or
468 unable to create one because apr_xlate_open returned APR_EINVAL, then
469 set *RET to null and return SVN_NO_ERROR; if fail for some other
470 reason, return error. */
472 get_uton_xlate_handle_node(xlate_handle_node_t **ret, apr_pool_t *pool)
474 return get_xlate_handle_node(ret,
475 assume_native_charset_is_utf8
476 ? SVN_APR_UTF8_CHARSET
477 : SVN_APR_LOCALE_CHARSET,
478 SVN_APR_UTF8_CHARSET,
479 SVN_UTF_UTON_XLATE_HANDLE, pool);
483 /* Convert SRC_LENGTH bytes of SRC_DATA in NODE->handle, store the result
484 in *DEST, which is allocated in POOL. */
486 convert_to_stringbuf(xlate_handle_node_t *node,
487 const char *src_data,
488 apr_size_t src_length,
489 svn_stringbuf_t **dest,
493 apr_status_t apr_err;
495 apr_err = svn_subr__win32_xlate_to_stringbuf(node->handle, src_data,
496 src_length, dest, pool);
498 apr_size_t buflen = src_length * 2;
499 apr_status_t apr_err;
500 apr_size_t srclen = src_length;
501 apr_size_t destlen = buflen;
503 /* Initialize *DEST to an empty stringbuf.
504 A 1:2 ratio of input bytes to output bytes (as assigned above)
505 should be enough for most translations, and if it turns out not
506 to be enough, we'll grow the buffer again, sizing it based on a
507 1:3 ratio of the remainder of the string. */
508 *dest = svn_stringbuf_create_ensure(buflen + 1, pool);
510 /* Not only does it not make sense to convert an empty string, but
511 apr-iconv is quite unreasonable about not allowing that. */
517 /* Set up state variables for xlate. */
518 destlen = buflen - (*dest)->len;
520 /* Attempt the conversion. */
521 apr_err = apr_xlate_conv_buffer(node->handle,
522 src_data + (src_length - srclen),
524 (*dest)->data + (*dest)->len,
527 /* Now, update the *DEST->len to track the amount of output data
528 churned out so far from this loop. */
529 (*dest)->len += ((buflen - (*dest)->len) - destlen);
530 buflen += srclen * 3; /* 3 is middle ground, 2 wasn't enough
531 for all characters in the buffer, 4 is
532 maximum character size (currently) */
535 } while (apr_err == APR_SUCCESS && srclen != 0);
538 /* If we exited the loop with an error, return the error. */
544 /* Can't use svn_error_wrap_apr here because it calls functions in
545 this file, leading to infinite recursion. */
546 if (node->frompage == SVN_APR_LOCALE_CHARSET)
547 errstr = apr_psprintf
548 (pool, _("Can't convert string from native encoding to '%s':"),
550 else if (node->topage == SVN_APR_LOCALE_CHARSET)
551 errstr = apr_psprintf
552 (pool, _("Can't convert string from '%s' to native encoding:"),
555 errstr = apr_psprintf
556 (pool, _("Can't convert string from '%s' to '%s':"),
557 node->frompage, node->topage);
559 err = svn_error_create(
560 apr_err, NULL, svn_utf__fuzzy_escape(src_data, src_length, pool));
561 return svn_error_create(apr_err, err, errstr);
563 /* Else, exited due to success. Trim the result buffer down to the
565 (*dest)->data[(*dest)->len] = '\0';
571 /* Return APR_EINVAL if the first LEN bytes of DATA contain anything
572 other than seven-bit, non-control (except for whitespace) ASCII
573 characters, finding the error pool from POOL. Otherwise, return
576 check_non_ascii(const char *data, apr_size_t len, apr_pool_t *pool)
578 const char *data_start = data;
580 for (; len > 0; --len, data++)
582 if ((! svn_ctype_isascii(*data))
583 || ((! svn_ctype_isspace(*data))
584 && svn_ctype_iscntrl(*data)))
586 /* Show the printable part of the data, followed by the
587 decimal code of the questionable character. Because if a
588 user ever gets this error, she's going to have to spend
589 time tracking down the non-ASCII data, so we want to help
590 as much as possible. And yes, we just call the unsafe
591 data "non-ASCII", even though the actual constraint is
592 somewhat more complex than that. */
594 if (data - data_start)
596 const char *error_data
597 = apr_pstrndup(pool, data_start, (data - data_start));
599 return svn_error_createf
601 _("Safe data '%s' was followed by non-ASCII byte %d: "
602 "unable to convert to/from UTF-8"),
603 error_data, *((const unsigned char *) data));
607 return svn_error_createf
609 _("Non-ASCII character (code %d) detected, "
610 "and unable to convert to/from UTF-8"),
611 *((const unsigned char *) data));
619 /* Construct an error with code APR_EINVAL and with a suitable message
620 * to describe the invalid UTF-8 sequence DATA of length LEN (which
621 * may have embedded NULLs). We can't simply print the data, almost
622 * by definition we don't really know how it is encoded.
625 invalid_utf8(const char *data, apr_size_t len, apr_pool_t *pool)
627 const char *last = svn_utf__last_valid(data, len);
628 const char *valid_txt = "", *invalid_txt = "";
630 size_t valid, invalid;
632 /* We will display at most 24 valid octets (this may split a leading
633 multi-byte character) as that should fit on one 80 character line. */
637 for (i = 0; i < valid; ++i)
638 valid_txt = apr_pstrcat(pool, valid_txt,
639 apr_psprintf(pool, " %02x",
640 (unsigned char)last[i-valid]),
643 /* 4 invalid octets will guarantee that the faulty octet is displayed */
644 invalid = data + len - last;
647 for (i = 0; i < invalid; ++i)
648 invalid_txt = apr_pstrcat(pool, invalid_txt,
649 apr_psprintf(pool, " %02x",
650 (unsigned char)last[i]),
653 return svn_error_createf(APR_EINVAL, NULL,
654 _("Valid UTF-8 data\n(hex:%s)\n"
655 "followed by invalid UTF-8 sequence\n(hex:%s)"),
656 valid_txt, invalid_txt);
659 /* Verify that the sequence DATA of length LEN is valid UTF-8.
660 If it is not, return an error with code APR_EINVAL. */
662 check_utf8(const char *data, apr_size_t len, apr_pool_t *pool)
664 if (! svn_utf__is_valid(data, len))
665 return invalid_utf8(data, len, pool);
669 /* Verify that the NULL terminated sequence DATA is valid UTF-8.
670 If it is not, return an error with code APR_EINVAL. */
672 check_cstring_utf8(const char *data, apr_pool_t *pool)
675 if (! svn_utf__cstring_is_valid(data))
676 return invalid_utf8(data, strlen(data), pool);
682 svn_utf_stringbuf_to_utf8(svn_stringbuf_t **dest,
683 const svn_stringbuf_t *src,
686 xlate_handle_node_t *node;
689 SVN_ERR(get_ntou_xlate_handle_node(&node, pool));
693 err = convert_to_stringbuf(node, src->data, src->len, dest, pool);
695 err = check_utf8((*dest)->data, (*dest)->len, pool);
699 err = check_non_ascii(src->data, src->len, pool);
701 *dest = svn_stringbuf_dup(src, pool);
704 return svn_error_compose_create(err,
705 put_xlate_handle_node
707 SVN_UTF_NTOU_XLATE_HANDLE,
713 svn_utf_string_to_utf8(const svn_string_t **dest,
714 const svn_string_t *src,
717 svn_stringbuf_t *destbuf;
718 xlate_handle_node_t *node;
721 SVN_ERR(get_ntou_xlate_handle_node(&node, pool));
725 err = convert_to_stringbuf(node, src->data, src->len, &destbuf, pool);
727 err = check_utf8(destbuf->data, destbuf->len, pool);
729 *dest = svn_stringbuf__morph_into_string(destbuf);
733 err = check_non_ascii(src->data, src->len, pool);
735 *dest = svn_string_dup(src, pool);
738 return svn_error_compose_create(err,
739 put_xlate_handle_node
741 SVN_UTF_NTOU_XLATE_HANDLE,
746 /* Common implementation for svn_utf_cstring_to_utf8,
747 svn_utf_cstring_to_utf8_ex, svn_utf_cstring_from_utf8 and
748 svn_utf_cstring_from_utf8_ex. Convert SRC to DEST using NODE->handle as
749 the translator and allocating from POOL. */
751 convert_cstring(const char **dest,
753 xlate_handle_node_t *node,
758 svn_stringbuf_t *destbuf;
759 SVN_ERR(convert_to_stringbuf(node, src, strlen(src),
761 *dest = destbuf->data;
765 apr_size_t len = strlen(src);
766 SVN_ERR(check_non_ascii(src, len, pool));
767 *dest = apr_pstrmemdup(pool, src, len);
774 svn_utf_cstring_to_utf8(const char **dest,
778 xlate_handle_node_t *node;
781 SVN_ERR(get_ntou_xlate_handle_node(&node, pool));
782 err = convert_cstring(dest, src, node, pool);
783 SVN_ERR(svn_error_compose_create(err,
784 put_xlate_handle_node
786 SVN_UTF_NTOU_XLATE_HANDLE,
788 return check_cstring_utf8(*dest, pool);
793 svn_utf_cstring_to_utf8_ex2(const char **dest,
795 const char *frompage,
798 xlate_handle_node_t *node;
800 const char *convset_key = get_xlate_key(SVN_APR_UTF8_CHARSET, frompage,
803 SVN_ERR(get_xlate_handle_node(&node, SVN_APR_UTF8_CHARSET, frompage,
805 err = convert_cstring(dest, src, node, pool);
806 SVN_ERR(svn_error_compose_create(err,
807 put_xlate_handle_node
809 SVN_UTF_NTOU_XLATE_HANDLE,
812 return check_cstring_utf8(*dest, pool);
817 svn_utf_cstring_to_utf8_ex(const char **dest,
819 const char *frompage,
820 const char *convset_key,
823 return svn_utf_cstring_to_utf8_ex2(dest, src, frompage, pool);
828 svn_utf_stringbuf_from_utf8(svn_stringbuf_t **dest,
829 const svn_stringbuf_t *src,
832 xlate_handle_node_t *node;
835 SVN_ERR(get_uton_xlate_handle_node(&node, pool));
839 err = check_utf8(src->data, src->len, pool);
841 err = convert_to_stringbuf(node, src->data, src->len, dest, pool);
845 err = check_non_ascii(src->data, src->len, pool);
847 *dest = svn_stringbuf_dup(src, pool);
850 err = svn_error_compose_create(
852 put_xlate_handle_node(node, SVN_UTF_UTON_XLATE_HANDLE, pool));
859 svn_utf_string_from_utf8(const svn_string_t **dest,
860 const svn_string_t *src,
863 svn_stringbuf_t *dbuf;
864 xlate_handle_node_t *node;
867 SVN_ERR(get_uton_xlate_handle_node(&node, pool));
871 err = check_utf8(src->data, src->len, pool);
873 err = convert_to_stringbuf(node, src->data, src->len,
876 *dest = svn_stringbuf__morph_into_string(dbuf);
880 err = check_non_ascii(src->data, src->len, pool);
882 *dest = svn_string_dup(src, pool);
885 err = svn_error_compose_create(
887 put_xlate_handle_node(node, SVN_UTF_UTON_XLATE_HANDLE, pool));
894 svn_utf_cstring_from_utf8(const char **dest,
898 xlate_handle_node_t *node;
901 SVN_ERR(check_cstring_utf8(src, pool));
903 SVN_ERR(get_uton_xlate_handle_node(&node, pool));
904 err = convert_cstring(dest, src, node, pool);
905 err = svn_error_compose_create(
907 put_xlate_handle_node(node, SVN_UTF_UTON_XLATE_HANDLE, pool));
914 svn_utf_cstring_from_utf8_ex2(const char **dest,
919 xlate_handle_node_t *node;
921 const char *convset_key = get_xlate_key(topage, SVN_APR_UTF8_CHARSET,
924 SVN_ERR(check_cstring_utf8(src, pool));
926 SVN_ERR(get_xlate_handle_node(&node, topage, SVN_APR_UTF8_CHARSET,
928 err = convert_cstring(dest, src, node, pool);
929 err = svn_error_compose_create(
931 put_xlate_handle_node(node, convset_key, pool));
937 svn_utf__cstring_from_utf8_fuzzy(const char *src,
939 svn_error_t *(*convert_from_utf8)
940 (const char **, const char *, apr_pool_t *))
942 const char *escaped, *converted;
945 escaped = svn_utf__fuzzy_escape(src, strlen(src), pool);
947 /* Okay, now we have a *new* UTF-8 string, one that's guaranteed to
948 contain only 7-bit bytes :-). Recode to native... */
949 err = convert_from_utf8(((const char **) &converted), escaped, pool);
953 svn_error_clear(err);
959 /* ### Check the client locale, maybe we can avoid that second
960 * conversion! See Ulrich Drepper's patch at
961 * http://subversion.tigris.org/issues/show_bug.cgi?id=807.
967 svn_utf_cstring_from_utf8_fuzzy(const char *src,
970 return svn_utf__cstring_from_utf8_fuzzy(src, pool,
971 svn_utf_cstring_from_utf8);
976 svn_utf_cstring_from_utf8_stringbuf(const char **dest,
977 const svn_stringbuf_t *src,
980 svn_stringbuf_t *destbuf;
982 SVN_ERR(svn_utf_stringbuf_from_utf8(&destbuf, src, pool));
983 *dest = destbuf->data;
990 svn_utf_cstring_from_utf8_string(const char **dest,
991 const svn_string_t *src,
994 svn_stringbuf_t *dbuf;
995 xlate_handle_node_t *node;
998 SVN_ERR(get_uton_xlate_handle_node(&node, pool));
1002 err = check_utf8(src->data, src->len, pool);
1004 err = convert_to_stringbuf(node, src->data, src->len,
1011 err = check_non_ascii(src->data, src->len, pool);
1013 *dest = apr_pstrmemdup(pool, src->data, src->len);
1016 err = svn_error_compose_create(
1018 put_xlate_handle_node(node, SVN_UTF_UTON_XLATE_HANDLE, pool));
1024 /* Insert the given UCS-4 VALUE into BUF at the given OFFSET. */
1026 membuf_insert_ucs4(svn_membuf_t *buf, apr_size_t offset, apr_int32_t value)
1028 svn_membuf__resize(buf, (offset + 1) * sizeof(value));
1029 ((apr_int32_t*)buf->data)[offset] = value;
1032 /* TODO: Use compiler intrinsics for byte swaps. */
1033 #define SWAP_SHORT(x) ((((x) & 0xff) << 8) | (((x) >> 8) & 0xff))
1034 #define SWAP_LONG(x) ((((x) & 0xff) << 24) | (((x) & 0xff00) << 8) \
1035 | (((x) >> 8) & 0xff00) | (((x) >> 24) & 0xff))
1037 #define IS_UTF16_LEAD_SURROGATE(c) ((c) >= 0xd800 && (c) <= 0xdbff)
1038 #define IS_UTF16_TRAIL_SURROGATE(c) ((c) >= 0xdc00 && (c) <= 0xdfff)
1041 svn_utf__utf16_to_utf8(const svn_string_t **result,
1042 const apr_uint16_t *utf16str,
1043 apr_size_t utf16len,
1044 svn_boolean_t big_endian,
1045 apr_pool_t *result_pool,
1046 apr_pool_t *scratch_pool)
1048 static const apr_uint16_t endiancheck = 0xa55a;
1049 const svn_boolean_t arch_big_endian =
1050 (((const char*)&endiancheck)[sizeof(endiancheck) - 1] == '\x5a');
1051 const svn_boolean_t swap_order = (!big_endian != !arch_big_endian);
1053 apr_uint16_t lead_surrogate;
1056 svn_membuf_t ucs4buf;
1057 svn_membuf_t resultbuf;
1060 if (utf16len == SVN_UTF__UNKNOWN_LENGTH)
1062 const apr_uint16_t *endp = utf16str;
1065 utf16len = (endp - utf16str);
1068 svn_membuf__create(&ucs4buf, utf16len * sizeof(apr_int32_t), scratch_pool);
1070 for (lead_surrogate = 0, length = 0, offset = 0;
1071 offset < utf16len; ++offset)
1073 const apr_uint16_t code =
1074 (swap_order ? SWAP_SHORT(utf16str[offset]) : utf16str[offset]);
1078 if (IS_UTF16_TRAIL_SURROGATE(code))
1080 /* Combine the lead and trail currogates into a 32-bit code. */
1081 membuf_insert_ucs4(&ucs4buf, length++,
1083 + (((lead_surrogate & 0x03ff) << 10)
1084 | (code & 0x03ff))));
1090 /* If we didn't find a surrogate pair, just dump the
1091 lead surrogate into the stream. */
1092 membuf_insert_ucs4(&ucs4buf, length++, lead_surrogate);
1097 if ((offset + 1) < utf16len && IS_UTF16_LEAD_SURROGATE(code))
1099 /* Store a lead surrogate that is followed by at least one
1100 code for the next iteration. */
1101 lead_surrogate = code;
1105 membuf_insert_ucs4(&ucs4buf, length++, code);
1108 /* Convert the UCS-4 buffer to UTF-8, assuming an average of 2 bytes
1109 per code point for encoding. The buffer will grow as
1111 svn_membuf__create(&resultbuf, length * 2, result_pool);
1112 SVN_ERR(svn_utf__encode_ucs4_string(
1113 &resultbuf, ucs4buf.data, length, &length));
1115 res = apr_palloc(result_pool, sizeof(*res));
1116 res->data = resultbuf.data;
1119 return SVN_NO_ERROR;
1124 svn_utf__utf32_to_utf8(const svn_string_t **result,
1125 const apr_int32_t *utf32str,
1126 apr_size_t utf32len,
1127 svn_boolean_t big_endian,
1128 apr_pool_t *result_pool,
1129 apr_pool_t *scratch_pool)
1131 static const apr_int32_t endiancheck = 0xa5cbbc5a;
1132 const svn_boolean_t arch_big_endian =
1133 (((const char*)&endiancheck)[sizeof(endiancheck) - 1] == '\x5a');
1134 const svn_boolean_t swap_order = (!big_endian != !arch_big_endian);
1137 svn_membuf_t resultbuf;
1140 if (utf32len == SVN_UTF__UNKNOWN_LENGTH)
1142 const apr_int32_t *endp = utf32str;
1145 utf32len = (endp - utf32str);
1151 svn_membuf_t ucs4buf;
1153 svn_membuf__create(&ucs4buf, utf32len * sizeof(apr_int32_t),
1156 for (offset = 0; offset < utf32len; ++offset)
1158 const apr_int32_t code = SWAP_LONG(utf32str[offset]);
1159 membuf_insert_ucs4(&ucs4buf, offset, code);
1161 utf32str = ucs4buf.data;
1164 /* Convert the UCS-4 buffer to UTF-8, assuming an average of 2 bytes
1165 per code point for encoding. The buffer will grow as
1167 svn_membuf__create(&resultbuf, utf32len * 2, result_pool);
1168 SVN_ERR(svn_utf__encode_ucs4_string(
1169 &resultbuf, utf32str, utf32len, &length));
1171 res = apr_palloc(result_pool, sizeof(*res));
1172 res->data = resultbuf.data;
1175 return SVN_NO_ERROR;
1183 svn_utf__win32_utf8_to_utf16(const WCHAR **result,
1185 const WCHAR *prefix,
1186 apr_pool_t *result_pool)
1188 const int utf8_count = strlen(src);
1189 const int prefix_len = (prefix ? lstrlenW(prefix) : 0);
1193 if (0 == prefix_len + utf8_count)
1196 return SVN_NO_ERROR;
1199 wide_count = MultiByteToWideChar(CP_UTF8, 0, src, utf8_count, NULL, 0);
1200 if (wide_count == 0)
1201 return svn_error_wrap_apr(apr_get_os_error(),
1202 _("Conversion to UTF-16 failed"));
1204 wide_str = apr_palloc(result_pool,
1205 (prefix_len + wide_count + 1) * sizeof(*wide_str));
1207 memcpy(wide_str, prefix, prefix_len * sizeof(*wide_str));
1208 if (0 == MultiByteToWideChar(CP_UTF8, 0, src, utf8_count,
1209 wide_str + prefix_len, wide_count))
1210 return svn_error_wrap_apr(apr_get_os_error(),
1211 _("Conversion to UTF-16 failed"));
1213 wide_str[prefix_len + wide_count] = 0;
1216 return SVN_NO_ERROR;
1220 svn_utf__win32_utf16_to_utf8(const char **result,
1223 apr_pool_t *result_pool)
1225 const int wide_count = lstrlenW(src);
1226 const int prefix_len = (prefix ? strlen(prefix) : 0);
1230 if (0 == prefix_len + wide_count)
1233 return SVN_NO_ERROR;
1236 utf8_count = WideCharToMultiByte(CP_UTF8, 0, src, wide_count,
1237 NULL, 0, NULL, FALSE);
1238 if (utf8_count == 0)
1239 return svn_error_wrap_apr(apr_get_os_error(),
1240 _("Conversion from UTF-16 failed"));
1242 utf8_str = apr_palloc(result_pool,
1243 (prefix_len + utf8_count + 1) * sizeof(*utf8_str));
1245 memcpy(utf8_str, prefix, prefix_len * sizeof(*utf8_str));
1246 if (0 == WideCharToMultiByte(CP_UTF8, 0, src, wide_count,
1247 utf8_str + prefix_len, utf8_count,
1249 return svn_error_wrap_apr(apr_get_os_error(),
1250 _("Conversion from UTF-16 failed"));
1252 utf8_str[prefix_len + utf8_count] = 0;
1255 return SVN_NO_ERROR;