2 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
3 * Copyright 2012 Garrett D'Amore <garrett@damore.org> All rights reserved.
4 * Copyright 2015 John Marino <draco@marino.st>
6 * This source code is derived from the illumos localedef command, and
7 * provided under BSD-style license terms by Nexenta Systems, Inc.
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
33 * The functions in this file convert from the standard multibyte forms
34 * to the wide character forms used internally by libc. Unfortunately,
35 * this approach means that we need a method for each and every encoding.
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
44 #include <sys/types.h>
45 #include "localedef.h"
47 static int towide_none(wchar_t *, const char *, unsigned);
48 static int towide_utf8(wchar_t *, const char *, unsigned);
49 static int towide_big5(wchar_t *, const char *, unsigned);
50 static int towide_gbk(wchar_t *, const char *, unsigned);
51 static int towide_gb2312(wchar_t *, const char *, unsigned);
52 static int towide_gb18030(wchar_t *, const char *, unsigned);
53 static int towide_mskanji(wchar_t *, const char *, unsigned);
54 static int towide_euccn(wchar_t *, const char *, unsigned);
55 static int towide_eucjp(wchar_t *, const char *, unsigned);
56 static int towide_euckr(wchar_t *, const char *, unsigned);
57 static int towide_euctw(wchar_t *, const char *, unsigned);
59 static int tomb_none(char *, wchar_t);
60 static int tomb_utf8(char *, wchar_t);
61 static int tomb_mbs(char *, wchar_t);
63 static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none;
64 static int (*_tomb)(char *, wchar_t) = tomb_none;
65 static char _encoding_buffer[20] = {'N','O','N','E'};
66 static const char *_encoding = _encoding_buffer;
67 static int _nbits = 7;
70 * Table of supported encodings. We only bother to list the multibyte
71 * encodings here, because single byte locales are handed by "NONE".
75 /* the name that the underlying libc implemenation uses */
77 /* the maximum number of bits required for priorities */
79 int (*towide)(wchar_t *, const char *, unsigned);
80 int (*tomb)(char *, wchar_t);
83 * UTF8 values max out at 0x1fffff (although in theory there could
84 * be later extensions, but it won't happen.) This means we only need
85 * 21 bits to be able to encode the entire range of priorities.
87 { "UTF-8", "UTF-8", 21, towide_utf8, tomb_utf8 },
88 { "UTF8", "UTF-8", 21, towide_utf8, tomb_utf8 },
89 { "utf8", "UTF-8", 21, towide_utf8, tomb_utf8 },
90 { "utf-8", "UTF-8", 21, towide_utf8, tomb_utf8 },
92 { "EUC-CN", "EUC-CN", 16, towide_euccn, tomb_mbs },
93 { "eucCN", "EUC-CN", 16, towide_euccn, tomb_mbs },
95 * Because the 3-byte form of EUC-JP use the same leading byte,
96 * only 17 bits required to provide unique priorities. (The low
97 * bit of that first byte is set.) By setting this value low,
98 * we can get by with only 3 bytes in the strxfrm expansion.
100 { "EUC-JP", "EUC-JP", 17, towide_eucjp, tomb_mbs },
101 { "eucJP", "EUC-JP", 17, towide_eucjp, tomb_mbs },
103 { "EUC-KR", "EUC-KR", 16, towide_euckr, tomb_mbs },
104 { "eucKR", "EUC-KR", 16, towide_euckr, tomb_mbs },
106 * EUC-TW uses 2 bytes most of the time, but 4 bytes if the
107 * high order byte is 0x8E. However, with 4 byte encodings,
108 * the third byte will be A0-B0. So we only need to consider
109 * the lower order 24 bits for collation.
111 { "EUC-TW", "EUC-TW", 24, towide_euctw, tomb_mbs },
112 { "eucTW", "EUC-TW", 24, towide_euctw, tomb_mbs },
114 { "MS_Kanji", "MSKanji", 16, towide_mskanji, tomb_mbs },
115 { "MSKanji", "MSKanji", 16, towide_mskanji, tomb_mbs },
116 { "PCK", "MSKanji", 16, towide_mskanji, tomb_mbs },
117 { "SJIS", "MSKanji", 16, towide_mskanji, tomb_mbs },
118 { "Shift_JIS", "MSKanji", 16, towide_mskanji, tomb_mbs },
120 { "BIG5", "BIG5", 16, towide_big5, tomb_mbs },
121 { "big5", "BIG5", 16, towide_big5, tomb_mbs },
122 { "Big5", "BIG5", 16, towide_big5, tomb_mbs },
124 { "GBK", "GBK", 16, towide_gbk, tomb_mbs },
127 * GB18030 can get away with just 31 bits. This is because the
128 * high order bit is always set for 4 byte values, and the
129 * at least one of the other bits in that 4 byte value will
132 { "GB18030", "GB18030", 31, towide_gb18030, tomb_mbs },
135 * This should probably be an aliase for euc-cn, or vice versa.
137 { "GB2312", "GB2312", 16, towide_gb2312, tomb_mbs },
139 { NULL, NULL, 0, 0, 0 },
143 show_mb(const char *mb)
147 /* ASCII stuff we just print */
148 if (isascii(*mb) && isgraph(*mb)) {
156 (void) snprintf(scr, sizeof (scr), "\\x%02x", *mb);
157 (void) strlcat(buf, scr, sizeof (buf));
163 static char *widemsg;
166 werr(const char *fmt, ...)
172 (void) vasprintf(&msg, fmt, va);
180 * This is used for 8-bit encodings.
183 towide_none(wchar_t *c, const char *mb, unsigned n __unused)
185 if (mb_cur_max != 1) {
186 werr("invalid or unsupported multibyte locale");
194 tomb_none(char *mb, wchar_t wc)
196 if (mb_cur_max != 1) {
197 werr("invalid or unsupported multibyte locale");
200 *(uint8_t *)mb = (wc & 0xff);
206 * UTF-8 stores wide characters in UTF-32 form.
209 towide_utf8(wchar_t *wc, const char *mb, unsigned n)
213 wchar_t lv; /* lowest legal value */
215 const uint8_t *s = (const uint8_t *)mb;
219 if ((c & 0x80) == 0) {
223 } else if ((c & 0xe0) == 0xc0) {
224 /* u80-u7ff - two bytes encoded */
228 } else if ((c & 0xf0) == 0xe0) {
229 /* u800-uffff - three bytes encoded */
233 } else if ((c & 0xf8) == 0xf0) {
234 /* u1000-u1fffff - four bytes encoded */
239 /* 5 and 6 byte encodings are not legal unicode */
240 werr("utf8 encoding too large (%s)", show_mb(mb));
244 werr("incomplete utf8 sequence (%s)", show_mb(mb));
248 for (i = 1; i < nb; i++) {
249 if (((s[i]) & 0xc0) != 0x80) {
250 werr("illegal utf8 byte (%x)", s[i]);
258 werr("illegal redundant utf8 encoding (%s)", show_mb(mb));
266 tomb_utf8(char *mb, wchar_t wc)
268 uint8_t *s = (uint8_t *)mb;
281 } else if (wc <= 0xffff) {
284 } else if (wc <= 0x1fffff) {
288 werr("illegal uf8 char (%x)", wc);
291 for (i = cnt - 1; i; i--) {
292 s[i] = (wc & 0x3f) | 0x80;
301 * Several encodings share a simplistic dual byte encoding. In these
302 * forms, they all indicate that a two byte sequence is to be used if
303 * the first byte has its high bit set. They all store this simple
304 * encoding as a 16-bit value, although a great many of the possible
305 * code points are not used in most character sets. This gives a possible
306 * set of just over 32,000 valid code points.
308 * 0x00 - 0x7f - 1 byte encoding
309 * 0x80 - 0x7fff - illegal
310 * 0x8000 - 0xffff - 2 byte encoding
314 towide_dbcs(wchar_t *wc, const char *mb, unsigned n)
318 c = *(const uint8_t *)mb;
320 if ((c & 0x80) == 0) {
326 werr("incomplete character sequence (%s)", show_mb(mb));
330 /* Store both bytes as a single 16-bit wide. */
332 c |= (uint8_t)(mb[1]);
338 * Most multibyte locales just convert the wide character to the multibyte
339 * form by stripping leading null bytes, and writing the 32-bit quantity
340 * in big-endian order.
343 tomb_mbs(char *mb, wchar_t wc)
345 uint8_t *s = (uint8_t *)mb;
348 if ((wc & 0xff000000U) != 0) {
350 } else if ((wc & 0x00ff0000U) != 0) {
352 } else if ((wc & 0x0000ff00U) != 0) {
363 /* ensure null termination */
370 * big5 is a simple dual byte character set.
373 towide_big5(wchar_t *wc, const char *mb, unsigned n)
375 return (towide_dbcs(wc, mb, n));
379 * GBK encodes wides in the same way that big5 does, the high order
380 * bit of the first byte indicates a double byte character.
383 towide_gbk(wchar_t *wc, const char *mb, unsigned n)
385 return (towide_dbcs(wc, mb, n));
389 * GB2312 is another DBCS. Its cleaner than others in that the second
390 * byte does not encode ASCII, but it supports characters.
393 towide_gb2312(wchar_t *wc, const char *mb, unsigned n)
395 return (towide_dbcs(wc, mb, n));
399 * GB18030. This encodes as 8, 16, or 32-bits.
400 * 7-bit values are in 1 byte, 4 byte sequences are used when
401 * the second byte encodes 0x30-39 and all other sequences are 2 bytes.
404 towide_gb18030(wchar_t *wc, const char *mb, unsigned n)
408 c = *(const uint8_t *)mb;
410 if ((c & 0x80) == 0) {
416 werr("incomplete character sequence (%s)", show_mb(mb));
420 /* pull in the second byte */
422 c |= (uint8_t)(mb[1]);
424 if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) {
426 werr("incomplete 4-byte character sequence (%s)",
431 c |= (uint8_t)(mb[2]);
433 c |= (uint8_t)(mb[3]);
443 * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it
444 * also has a range of single byte characters above 0x80. (0xa1-0xdf).
447 towide_mskanji(wchar_t *wc, const char *mb, unsigned n)
451 c = *(const uint8_t *)mb;
453 if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) {
460 werr("incomplete character sequence (%s)", show_mb(mb));
464 /* Store both bytes as a single 16-bit wide. */
466 c |= (uint8_t)(mb[1]);
472 * EUC forms. EUC encodings are "variable". FreeBSD carries some additional
473 * variable data to encode these, but we're going to treat each as independent
474 * instead. Its the only way we can sensibly move forward.
476 * Note that the way in which the different EUC forms vary is how wide
477 * CS2 and CS3 are and what the first byte of them is.
480 towide_euc_impl(wchar_t *wc, const char *mb, unsigned n,
481 uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width)
487 c = *(const uint8_t *)mb;
490 * All variations of EUC encode 7-bit ASCII as one byte, and use
491 * additional bytes for more than that.
493 if ((c & 0x80) == 0) {
500 * All EUC variants reserve 0xa1-0xff to identify CS1, which
501 * is always two bytes wide. Note that unused CS will be zero,
502 * and that cannot be true because we know that the high order
507 } else if (c == cs2) {
509 } else if (c == cs3) {
513 if ((int)n < width) {
514 werr("incomplete character sequence (%s)", show_mb(mb));
518 for (i = 1; i < width; i++) {
519 /* pull in the next byte */
521 c |= (uint8_t)(mb[i]);
529 * EUC-CN encodes as follows:
531 * Code set 0 (ASCII): 0x21-0x7E
532 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE
537 towide_euccn(wchar_t *wc, const char *mb, unsigned n)
539 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
543 * EUC-JP encodes as follows:
545 * Code set 0 (ASCII or JIS X 0201-1976 Roman): 0x21-0x7E
546 * Code set 1 (JIS X 0208): 0xA1A1-0xFEFE
547 * Code set 2 (half-width katakana): 0x8EA1-0x8EDF
548 * Code set 3 (JIS X 0212-1990): 0x8FA1A1-0x8FFEFE
551 towide_eucjp(wchar_t *wc, const char *mb, unsigned n)
553 return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3));
557 * EUC-KR encodes as follows:
559 * Code set 0 (ASCII or KS C 5636-1993): 0x21-0x7E
560 * Code set 1 (KS C 5601-1992): 0xA1A1-0xFEFE
565 towide_euckr(wchar_t *wc, const char *mb, unsigned n)
567 return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0));
571 * EUC-TW encodes as follows:
573 * Code set 0 (ASCII): 0x21-0x7E
574 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE
575 * Code set 2 (CNS 11643-1992 Planes 1-16): 0x8EA1A1A1-0x8EB0FEFE
579 towide_euctw(wchar_t *wc, const char *mb, unsigned n)
581 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
585 * Public entry points.
589 to_wide(wchar_t *wc, const char *mb)
591 /* this won't fail hard */
592 return (_towide(wc, mb, strlen(mb)));
596 to_mb(char *mb, wchar_t wc)
600 if ((rv = _tomb(mb, wc)) < 0) {
609 to_mb_string(const wchar_t *wcs)
615 mbs = malloc((wcslen(wcs) * mb_cur_max) + 1);
617 warn("out of memory");
622 if ((len = to_mb(ptr, *wcs)) < 0) {
635 set_wide_encoding(const char *encoding)
639 _towide = towide_none;
643 snprintf(_encoding_buffer, sizeof(_encoding_buffer), "NONE:%s",
645 for (i = 0; mb_encodings[i].name; i++) {
646 if (strcasecmp(encoding, mb_encodings[i].name) == 0) {
647 _towide = mb_encodings[i].towide;
648 _tomb = mb_encodings[i].tomb;
649 _encoding = mb_encodings[i].cname;
650 _nbits = mb_encodings[i].nbits;
657 get_wide_encoding(void)
665 return ((int)((1U << _nbits) - 1));