4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
26 #pragma ident "%Z%%M% %I% %E% SMI"
30 * UTF-8 text preparation functions (PSARC/2007/149, PSARC/2007/458).
32 * Man pages: u8_textprep_open(9F), u8_textprep_buf(9F), u8_textprep_close(9F),
33 * u8_textprep_str(9F), u8_strcmp(9F), and u8_validate(9F). See also
34 * the section 3C man pages.
35 * Interface stability: Committed.
38 #include <sys/types.h>
40 #include <sys/param.h>
41 #include <sys/sysmacros.h>
42 #include <sys/systm.h>
43 #include <sys/debug.h>
48 #include <sys/byteorder.h>
49 #include <sys/errno.h>
50 #include <sys/u8_textprep.h>
51 #include <sys/u8_textprep_data.h>
54 /* The maximum possible number of bytes in a UTF-8 character. */
55 #define U8_MB_CUR_MAX (4)
58 * The maximum number of bytes needed for a UTF-8 character to cover
59 * U+0000 - U+FFFF, i.e., the coding space of now deprecated UCS-2.
61 #define U8_MAX_BYTES_UCS2 (3)
63 /* The maximum possible number of bytes in a Stream-Safe Text. */
64 #define U8_STREAM_SAFE_TEXT_MAX (128)
67 * The maximum number of characters in a combining/conjoining sequence and
68 * the actual upperbound limit of a combining/conjoining sequence.
70 #define U8_MAX_CHARS_A_SEQ (32)
71 #define U8_UPPER_LIMIT_IN_A_SEQ (31)
73 /* The combining class value for Starter. */
74 #define U8_COMBINING_CLASS_STARTER (0)
77 * Some Hangul related macros at below.
79 * The first and the last of Hangul syllables, Hangul Jamo Leading consonants,
80 * Vowels, and optional Trailing consonants in Unicode scalar values.
82 * Please be noted that the U8_HANGUL_JAMO_T_FIRST is 0x11A7 at below not
83 * the actual U+11A8. This is due to that the trailing consonant is optional
84 * and thus we are doing a pre-calculation of subtracting one.
86 * Each of 19 modern leading consonants has total 588 possible syllables since
87 * Hangul has 21 modern vowels and 27 modern trailing consonants plus 1 for
88 * no trailing consonant case, i.e., 21 x 28 = 588.
90 * We also have bunch of Hangul related macros at below. Please bear in mind
91 * that the U8_HANGUL_JAMO_1ST_BYTE can be used to check whether it is
92 * a Hangul Jamo or not but the value does not guarantee that it is a Hangul
93 * Jamo; it just guarantee that it will be most likely.
95 #define U8_HANGUL_SYL_FIRST (0xAC00U)
96 #define U8_HANGUL_SYL_LAST (0xD7A3U)
98 #define U8_HANGUL_JAMO_L_FIRST (0x1100U)
99 #define U8_HANGUL_JAMO_L_LAST (0x1112U)
100 #define U8_HANGUL_JAMO_V_FIRST (0x1161U)
101 #define U8_HANGUL_JAMO_V_LAST (0x1175U)
102 #define U8_HANGUL_JAMO_T_FIRST (0x11A7U)
103 #define U8_HANGUL_JAMO_T_LAST (0x11C2U)
105 #define U8_HANGUL_V_COUNT (21)
106 #define U8_HANGUL_VT_COUNT (588)
107 #define U8_HANGUL_T_COUNT (28)
109 #define U8_HANGUL_JAMO_1ST_BYTE (0xE1U)
111 #define U8_SAVE_HANGUL_AS_UTF8(s, i, j, k, b) \
112 (s)[(i)] = (uchar_t)(0xE0U | ((uint32_t)(b) & 0xF000U) >> 12); \
113 (s)[(j)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x0FC0U) >> 6); \
114 (s)[(k)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x003FU));
116 #define U8_HANGUL_JAMO_L(u) \
117 ((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_L_LAST)
119 #define U8_HANGUL_JAMO_V(u) \
120 ((u) >= U8_HANGUL_JAMO_V_FIRST && (u) <= U8_HANGUL_JAMO_V_LAST)
122 #define U8_HANGUL_JAMO_T(u) \
123 ((u) > U8_HANGUL_JAMO_T_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST)
125 #define U8_HANGUL_JAMO(u) \
126 ((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST)
128 #define U8_HANGUL_SYLLABLE(u) \
129 ((u) >= U8_HANGUL_SYL_FIRST && (u) <= U8_HANGUL_SYL_LAST)
131 #define U8_HANGUL_COMPOSABLE_L_V(s, u) \
132 ((s) == U8_STATE_HANGUL_L && U8_HANGUL_JAMO_V((u)))
134 #define U8_HANGUL_COMPOSABLE_LV_T(s, u) \
135 ((s) == U8_STATE_HANGUL_LV && U8_HANGUL_JAMO_T((u)))
137 /* The types of decomposition mappings. */
138 #define U8_DECOMP_BOTH (0xF5U)
139 #define U8_DECOMP_CANONICAL (0xF6U)
141 /* The indicator for 16-bit table. */
142 #define U8_16BIT_TABLE_INDICATOR (0x8000U)
144 /* The following are some convenience macros. */
145 #define U8_PUT_3BYTES_INTO_UTF32(u, b1, b2, b3) \
146 (u) = ((uint32_t)(b1) & 0x0F) << 12 | ((uint32_t)(b2) & 0x3F) << 6 | \
147 (uint32_t)(b3) & 0x3F;
149 #define U8_SIMPLE_SWAP(a, b, t) \
154 #define U8_ASCII_TOUPPER(c) \
155 (((c) >= 'a' && (c) <= 'z') ? (c) - 'a' + 'A' : (c))
157 #define U8_ASCII_TOLOWER(c) \
158 (((c) >= 'A' && (c) <= 'Z') ? (c) - 'A' + 'a' : (c))
160 #define U8_ISASCII(c) (((uchar_t)(c)) < 0x80U)
162 * The following macro assumes that the two characters that are to be
163 * swapped are adjacent to each other and 'a' comes before 'b'.
165 * If the assumptions are not met, then, the macro will fail.
167 #define U8_SWAP_COMB_MARKS(a, b) \
168 for (k = 0; k < disp[(a)]; k++) \
169 u8t[k] = u8s[start[(a)] + k]; \
170 for (k = 0; k < disp[(b)]; k++) \
171 u8s[start[(a)] + k] = u8s[start[(b)] + k]; \
172 start[(b)] = start[(a)] + disp[(b)]; \
173 for (k = 0; k < disp[(a)]; k++) \
174 u8s[start[(b)] + k] = u8t[k]; \
175 U8_SIMPLE_SWAP(comb_class[(a)], comb_class[(b)], tc); \
176 U8_SIMPLE_SWAP(disp[(a)], disp[(b)], tc);
178 /* The possible states during normalization. */
181 U8_STATE_HANGUL_L = 1,
182 U8_STATE_HANGUL_LV = 2,
183 U8_STATE_HANGUL_LVT = 3,
184 U8_STATE_HANGUL_V = 4,
185 U8_STATE_HANGUL_T = 5,
186 U8_STATE_COMBINING_MARK = 6
187 } u8_normalization_states_t;
190 * The three vectors at below are used to check bytes of a given UTF-8
191 * character are valid and not containing any malformed byte values.
193 * We used to have a quite relaxed UTF-8 binary representation but then there
194 * was some security related issues and so the Unicode Consortium defined
195 * and announced the UTF-8 Corrigendum at Unicode 3.1 and then refined it
196 * one more time at the Unicode 3.2. The following three tables are based on
200 #define U8_ILLEGAL_NEXT_BYTE_COMMON(c) ((c) < 0x80 || (c) > 0xBF)
202 #define I_ U8_ILLEGAL_CHAR
203 #define O_ U8_OUT_OF_RANGE_CHAR
205 const int8_t u8_number_of_bytes[0x100] = {
206 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
207 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
208 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
209 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
210 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
211 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
212 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
213 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
215 /* 80 81 82 83 84 85 86 87 88 89 8A 8B 8C 8D 8E 8F */
216 I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_,
218 /* 90 91 92 93 94 95 96 97 98 99 9A 9B 9C 9D 9E 9F */
219 I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_,
221 /* A0 A1 A2 A3 A4 A5 A6 A7 A8 A9 AA AB AC AD AE AF */
222 I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_,
224 /* B0 B1 B2 B3 B4 B5 B6 B7 B8 B9 BA BB BC BD BE BF */
225 I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_,
227 /* C0 C1 C2 C3 C4 C5 C6 C7 C8 C9 CA CB CC CD CE CF */
228 I_, I_, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
230 /* D0 D1 D2 D3 D4 D5 D6 D7 D8 D9 DA DB DC DD DE DF */
231 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
233 /* E0 E1 E2 E3 E4 E5 E6 E7 E8 E9 EA EB EC ED EE EF */
234 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
236 /* F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 FA FB FC FD FE FF */
237 4, 4, 4, 4, 4, O_, O_, O_, O_, O_, O_, O_, O_, O_, O_, O_,
243 const uint8_t u8_valid_min_2nd_byte[0x100] = {
244 0, 0, 0, 0, 0, 0, 0, 0,
245 0, 0, 0, 0, 0, 0, 0, 0,
246 0, 0, 0, 0, 0, 0, 0, 0,
247 0, 0, 0, 0, 0, 0, 0, 0,
248 0, 0, 0, 0, 0, 0, 0, 0,
249 0, 0, 0, 0, 0, 0, 0, 0,
250 0, 0, 0, 0, 0, 0, 0, 0,
251 0, 0, 0, 0, 0, 0, 0, 0,
252 0, 0, 0, 0, 0, 0, 0, 0,
253 0, 0, 0, 0, 0, 0, 0, 0,
254 0, 0, 0, 0, 0, 0, 0, 0,
255 0, 0, 0, 0, 0, 0, 0, 0,
256 0, 0, 0, 0, 0, 0, 0, 0,
257 0, 0, 0, 0, 0, 0, 0, 0,
258 0, 0, 0, 0, 0, 0, 0, 0,
259 0, 0, 0, 0, 0, 0, 0, 0,
260 0, 0, 0, 0, 0, 0, 0, 0,
261 0, 0, 0, 0, 0, 0, 0, 0,
262 0, 0, 0, 0, 0, 0, 0, 0,
263 0, 0, 0, 0, 0, 0, 0, 0,
264 0, 0, 0, 0, 0, 0, 0, 0,
265 0, 0, 0, 0, 0, 0, 0, 0,
266 0, 0, 0, 0, 0, 0, 0, 0,
267 0, 0, 0, 0, 0, 0, 0, 0,
268 /* C0 C1 C2 C3 C4 C5 C6 C7 */
269 0, 0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
270 /* C8 C9 CA CB CC CD CE CF */
271 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
272 /* D0 D1 D2 D3 D4 D5 D6 D7 */
273 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
274 /* D8 D9 DA DB DC DD DE DF */
275 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
276 /* E0 E1 E2 E3 E4 E5 E6 E7 */
277 0xa0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
278 /* E8 E9 EA EB EC ED EE EF */
279 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
280 /* F0 F1 F2 F3 F4 F5 F6 F7 */
281 0x90, 0x80, 0x80, 0x80, 0x80, 0, 0, 0,
282 0, 0, 0, 0, 0, 0, 0, 0,
285 const uint8_t u8_valid_max_2nd_byte[0x100] = {
286 0, 0, 0, 0, 0, 0, 0, 0,
287 0, 0, 0, 0, 0, 0, 0, 0,
288 0, 0, 0, 0, 0, 0, 0, 0,
289 0, 0, 0, 0, 0, 0, 0, 0,
290 0, 0, 0, 0, 0, 0, 0, 0,
291 0, 0, 0, 0, 0, 0, 0, 0,
292 0, 0, 0, 0, 0, 0, 0, 0,
293 0, 0, 0, 0, 0, 0, 0, 0,
294 0, 0, 0, 0, 0, 0, 0, 0,
295 0, 0, 0, 0, 0, 0, 0, 0,
296 0, 0, 0, 0, 0, 0, 0, 0,
297 0, 0, 0, 0, 0, 0, 0, 0,
298 0, 0, 0, 0, 0, 0, 0, 0,
299 0, 0, 0, 0, 0, 0, 0, 0,
300 0, 0, 0, 0, 0, 0, 0, 0,
301 0, 0, 0, 0, 0, 0, 0, 0,
302 0, 0, 0, 0, 0, 0, 0, 0,
303 0, 0, 0, 0, 0, 0, 0, 0,
304 0, 0, 0, 0, 0, 0, 0, 0,
305 0, 0, 0, 0, 0, 0, 0, 0,
306 0, 0, 0, 0, 0, 0, 0, 0,
307 0, 0, 0, 0, 0, 0, 0, 0,
308 0, 0, 0, 0, 0, 0, 0, 0,
309 0, 0, 0, 0, 0, 0, 0, 0,
310 /* C0 C1 C2 C3 C4 C5 C6 C7 */
311 0, 0, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
312 /* C8 C9 CA CB CC CD CE CF */
313 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
314 /* D0 D1 D2 D3 D4 D5 D6 D7 */
315 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
316 /* D8 D9 DA DB DC DD DE DF */
317 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
318 /* E0 E1 E2 E3 E4 E5 E6 E7 */
319 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
320 /* E8 E9 EA EB EC ED EE EF */
321 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0x9f, 0xbf, 0xbf,
322 /* F0 F1 F2 F3 F4 F5 F6 F7 */
323 0xbf, 0xbf, 0xbf, 0xbf, 0x8f, 0, 0, 0,
324 0, 0, 0, 0, 0, 0, 0, 0,
329 * The u8_validate() validates on the given UTF-8 character string and
330 * calculate the byte length. It is quite similar to mblen(3C) except that
331 * this will validate against the list of characters if required and
332 * specific to UTF-8 and Unicode.
335 u8_validate(char *u8str, size_t n, char **list, int flag, int *errnum)
347 boolean_t no_need_to_validate_entire;
348 boolean_t check_additional;
349 boolean_t validate_ucs2_range_only;
354 ib = (uchar_t *)u8str;
359 no_need_to_validate_entire = ! (flag & U8_VALIDATE_ENTIRE);
360 check_additional = flag & U8_VALIDATE_CHECK_ADDITIONAL;
361 validate_ucs2_range_only = flag & U8_VALIDATE_UCS2_RANGE;
363 while (ib < ibtail) {
365 * The first byte of a UTF-8 character tells how many
366 * bytes will follow for the character. If the first byte
367 * is an illegal byte value or out of range value, we just
368 * return -1 with an appropriate error number.
370 sz = u8_number_of_bytes[*ib];
371 if (sz == U8_ILLEGAL_CHAR) {
376 if (sz == U8_OUT_OF_RANGE_CHAR ||
377 (validate_ucs2_range_only && sz > U8_MAX_BYTES_UCS2)) {
383 * If we don't have enough bytes to check on, that's also
384 * an error. As you can see, we give illegal byte sequence
385 * checking higher priority then EINVAL cases.
387 if ((ibtail - ib) < sz) {
397 * Check on the multi-byte UTF-8 character. For more
398 * details on this, see comment added for the used
399 * data structures at the beginning of the file.
404 for (i = 1; i < sz; i++) {
406 if (*ib < u8_valid_min_2nd_byte[f] ||
407 *ib > u8_valid_max_2nd_byte[f]) {
412 } else if (U8_ILLEGAL_NEXT_BYTE_COMMON(*ib)) {
421 if (check_additional) {
422 for (p = (uchar_t **)list, i = 0; p[i]; i++) {
426 if (*s1 != *s2 || *s2 == '\0')
432 if (s1 >= ib && *s2 == '\0') {
439 if (no_need_to_validate_entire)
447 * The do_case_conv() looks at the mapping tables and returns found
448 * bytes if any. If not found, the input bytes are returned. The function
449 * always terminate the return bytes with a null character assuming that
450 * there are plenty of room to do so.
452 * The case conversions are simple case conversions mapping a character to
453 * another character as specified in the Unicode data. The byte size of
454 * the mapped character could be different from that of the input character.
456 * The return value is the byte length of the returned character excluding
457 * the terminating null byte.
460 do_case_conv(int uv, uchar_t *u8s, uchar_t *s, int sz, boolean_t is_it_toupper)
473 * At this point, the only possible values for sz are 2, 3, and 4.
474 * The u8s should point to a vector that is well beyond the size of
480 } else if (sz == 3) {
484 } else if (sz == 4) {
490 /* This is not possible but just in case as a fallback. */
492 *u8s = U8_ASCII_TOUPPER(*s);
494 *u8s = U8_ASCII_TOLOWER(*s);
502 * Let's find out if we have a corresponding character.
504 b1 = u8_common_b1_tbl[uv][b1];
505 if (b1 == U8_TBL_ELEMENT_NOT_DEF)
508 b2 = u8_case_common_b2_tbl[uv][b1][b2];
509 if (b2 == U8_TBL_ELEMENT_NOT_DEF)
513 b3_tbl = u8_toupper_b3_tbl[uv][b2][b3].tbl_id;
514 if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF)
517 start_id = u8_toupper_b4_tbl[uv][b3_tbl][b4];
518 end_id = u8_toupper_b4_tbl[uv][b3_tbl][b4 + 1];
520 /* Either there is no match or an error at the table. */
521 if (start_id >= end_id || (end_id - start_id) > U8_MB_CUR_MAX)
524 b3_base = u8_toupper_b3_tbl[uv][b2][b3].base;
526 for (i = 0; start_id < end_id; start_id++)
527 u8s[i++] = u8_toupper_final_tbl[uv][b3_base + start_id];
529 b3_tbl = u8_tolower_b3_tbl[uv][b2][b3].tbl_id;
530 if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF)
533 start_id = u8_tolower_b4_tbl[uv][b3_tbl][b4];
534 end_id = u8_tolower_b4_tbl[uv][b3_tbl][b4 + 1];
536 if (start_id >= end_id || (end_id - start_id) > U8_MB_CUR_MAX)
539 b3_base = u8_tolower_b3_tbl[uv][b2][b3].base;
541 for (i = 0; start_id < end_id; start_id++)
542 u8s[i++] = u8_tolower_final_tbl[uv][b3_base + start_id];
546 * If i is still zero, that means there is no corresponding character.
557 * The do_case_compare() function compares the two input strings, s1 and s2,
558 * one character at a time doing case conversions if applicable and return
559 * the comparison result as like strcmp().
561 * Since, in empirical sense, most of text data are 7-bit ASCII characters,
562 * we treat the 7-bit ASCII characters as a special case trying to yield
563 * faster processing time.
566 do_case_compare(size_t uv, uchar_t *s1, uchar_t *s2, size_t n1,
567 size_t n2, boolean_t is_it_toupper, int *errnum)
575 uchar_t u8s1[U8_MB_CUR_MAX + 1];
576 uchar_t u8s2[U8_MB_CUR_MAX + 1];
579 while (i1 < n1 && i2 < n2) {
581 * Find out what would be the byte length for this UTF-8
582 * character at string s1 and also find out if this is
583 * an illegal start byte or not and if so, issue a proper
584 * error number and yet treat this byte as a character.
586 sz1 = u8_number_of_bytes[*s1];
593 * For 7-bit ASCII characters mainly, we do a quick case
594 * conversion right at here.
596 * If we don't have enough bytes for this character, issue
597 * an EINVAL error and use what are available.
599 * If we have enough bytes, find out if there is
600 * a corresponding uppercase character and if so, copy over
601 * the bytes for a comparison later. If there is no
602 * corresponding uppercase character, then, use what we have
603 * for the comparison.
607 u8s1[0] = U8_ASCII_TOUPPER(*s1);
609 u8s1[0] = U8_ASCII_TOLOWER(*s1);
612 } else if ((i1 + sz1) > n1) {
614 for (j = 0; (i1 + j) < n1; )
618 (void) do_case_conv(uv, u8s1, s1, sz1, is_it_toupper);
622 /* Do the same for the string s2. */
623 sz2 = u8_number_of_bytes[*s2];
631 u8s2[0] = U8_ASCII_TOUPPER(*s2);
633 u8s2[0] = U8_ASCII_TOLOWER(*s2);
636 } else if ((i2 + sz2) > n2) {
638 for (j = 0; (i2 + j) < n2; )
642 (void) do_case_conv(uv, u8s2, s2, sz2, is_it_toupper);
646 /* Now compare the two characters. */
647 if (sz1 == 1 && sz2 == 1) {
653 f = strcmp((const char *)u8s1, (const char *)u8s2);
659 * They were the same. Let's move on to the next
667 * We compared until the end of either or both strings.
669 * If we reached to or went over the ends for the both, that means
672 * If we reached only one of the two ends, that means the other string
673 * has something which then the fact can be used to determine
685 * The combining_class() function checks on the given bytes and find out
686 * the corresponding Unicode combining class value. The return value 0 means
687 * it is a Starter. Any illegal UTF-8 character will also be treated as
691 combining_class(size_t uv, uchar_t *s, size_t sz)
698 if (sz == 1 || sz > 4)
704 } else if (sz == 3) {
708 } else if (sz == 4) {
715 b1 = u8_common_b1_tbl[uv][b1];
716 if (b1 == U8_TBL_ELEMENT_NOT_DEF)
719 b2 = u8_combining_class_b2_tbl[uv][b1][b2];
720 if (b2 == U8_TBL_ELEMENT_NOT_DEF)
723 b3 = u8_combining_class_b3_tbl[uv][b2][b3];
724 if (b3 == U8_TBL_ELEMENT_NOT_DEF)
727 return (u8_combining_class_b4_tbl[uv][b3][b4]);
731 * The do_decomp() function finds out a matching decomposition if any
732 * and return. If there is no match, the input bytes are copied and returned.
733 * The function also checks if there is a Hangul, decomposes it if necessary
736 * To save time, a single byte 7-bit ASCII character should be handled by
739 * The function returns the number of bytes returned sans always terminating
740 * the null byte. It will also return a state that will tell if there was
741 * a Hangul character decomposed which then will be used by the caller.
744 do_decomp(size_t uv, uchar_t *u8s, uchar_t *s, int sz,
745 boolean_t canonical_decomposition, u8_normalization_states_t *state)
762 } else if (sz == 3) {
763 /* Convert it to a Unicode scalar value. */
764 U8_PUT_3BYTES_INTO_UTF32(u1, s[0], s[1], s[2]);
767 * If this is a Hangul syllable, we decompose it into
768 * a leading consonant, a vowel, and an optional trailing
769 * consonant and then return.
771 if (U8_HANGUL_SYLLABLE(u1)) {
772 u1 -= U8_HANGUL_SYL_FIRST;
774 b1 = U8_HANGUL_JAMO_L_FIRST + u1 / U8_HANGUL_VT_COUNT;
775 b2 = U8_HANGUL_JAMO_V_FIRST + (u1 % U8_HANGUL_VT_COUNT)
777 b3 = u1 % U8_HANGUL_T_COUNT;
779 U8_SAVE_HANGUL_AS_UTF8(u8s, 0, 1, 2, b1);
780 U8_SAVE_HANGUL_AS_UTF8(u8s, 3, 4, 5, b2);
782 b3 += U8_HANGUL_JAMO_T_FIRST;
783 U8_SAVE_HANGUL_AS_UTF8(u8s, 6, 7, 8, b3);
786 *state = U8_STATE_HANGUL_LVT;
791 *state = U8_STATE_HANGUL_LV;
801 * If this is a Hangul Jamo, we know there is nothing
802 * further that we can decompose.
804 if (U8_HANGUL_JAMO_L(u1)) {
805 *state = U8_STATE_HANGUL_L;
809 if (U8_HANGUL_JAMO_V(u1)) {
810 if (*state == U8_STATE_HANGUL_L)
811 *state = U8_STATE_HANGUL_LV;
813 *state = U8_STATE_HANGUL_V;
817 if (U8_HANGUL_JAMO_T(u1)) {
818 if (*state == U8_STATE_HANGUL_LV)
819 *state = U8_STATE_HANGUL_LVT;
821 *state = U8_STATE_HANGUL_T;
824 } else if (sz == 4) {
832 * This is a fallback and should not happen if the function
833 * was called properly.
837 *state = U8_STATE_START;
842 * At this point, this rountine does not know what it would get.
843 * The caller should sort it out if the state isn't a Hangul one.
845 *state = U8_STATE_START;
847 /* Try to find matching decomposition mapping byte sequence. */
848 b1 = u8_common_b1_tbl[uv][b1];
849 if (b1 == U8_TBL_ELEMENT_NOT_DEF)
852 b2 = u8_decomp_b2_tbl[uv][b1][b2];
853 if (b2 == U8_TBL_ELEMENT_NOT_DEF)
856 b3_tbl = u8_decomp_b3_tbl[uv][b2][b3].tbl_id;
857 if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF)
861 * If b3_tbl is bigger than or equal to U8_16BIT_TABLE_INDICATOR
862 * which is 0x8000, this means we couldn't fit the mappings into
863 * the cardinality of a unsigned byte.
865 if (b3_tbl >= U8_16BIT_TABLE_INDICATOR) {
866 b3_tbl -= U8_16BIT_TABLE_INDICATOR;
867 start_id = u8_decomp_b4_16bit_tbl[uv][b3_tbl][b4];
868 end_id = u8_decomp_b4_16bit_tbl[uv][b3_tbl][b4 + 1];
870 start_id = u8_decomp_b4_tbl[uv][b3_tbl][b4];
871 end_id = u8_decomp_b4_tbl[uv][b3_tbl][b4 + 1];
874 /* This also means there wasn't any matching decomposition. */
875 if (start_id >= end_id)
879 * The final table for decomposition mappings has three types of
880 * byte sequences depending on whether a mapping is for compatibility
881 * decomposition, canonical decomposition, or both like the following:
883 * (1) Compatibility decomposition mappings:
885 * +---+---+-...-+---+
886 * | B0| B1| ... | Bm|
887 * +---+---+-...-+---+
889 * The first byte, B0, is always less then 0xF5 (U8_DECOMP_BOTH).
891 * (2) Canonical decomposition mappings:
893 * +---+---+---+-...-+---+
894 * | T | b0| b1| ... | bn|
895 * +---+---+---+-...-+---+
897 * where the first byte, T, is 0xF6 (U8_DECOMP_CANONICAL).
901 * +---+---+---+---+-...-+---+---+---+-...-+---+
902 * | T | D | b0| b1| ... | bn| B0| B1| ... | Bm|
903 * +---+---+---+---+-...-+---+---+---+-...-+---+
905 * where T is 0xF5 (U8_DECOMP_BOTH) and D is a displacement
906 * byte, b0 to bn are canonical mapping bytes and B0 to Bm are
907 * compatibility mapping bytes.
909 * Note that compatibility decomposition means doing recursive
910 * decompositions using both compatibility decomposition mappings and
911 * canonical decomposition mappings. On the other hand, canonical
912 * decomposition means doing recursive decompositions using only
913 * canonical decomposition mappings. Since the table we have has gone
914 * through the recursions already, we do not need to do so during
915 * runtime, i.e., the table has been completely flattened out
919 b3_base = u8_decomp_b3_tbl[uv][b2][b3].base;
921 /* Get the type, T, of the byte sequence. */
922 b1 = u8_decomp_final_tbl[uv][b3_base + start_id];
925 * If necessary, adjust start_id, end_id, or both. Note that if
926 * this is compatibility decomposition mapping, there is no
929 if (canonical_decomposition) {
930 /* Is the mapping only for compatibility decomposition? */
931 if (b1 < U8_DECOMP_BOTH)
936 if (b1 == U8_DECOMP_BOTH) {
938 u8_decomp_final_tbl[uv][b3_base + start_id];
943 * Unless this is a compatibility decomposition mapping,
944 * we adjust the start_id.
946 if (b1 == U8_DECOMP_BOTH) {
948 start_id += u8_decomp_final_tbl[uv][b3_base + start_id];
949 } else if (b1 == U8_DECOMP_CANONICAL) {
954 for (i = 0; start_id < end_id; start_id++)
955 u8s[i++] = u8_decomp_final_tbl[uv][b3_base + start_id];
962 * The find_composition_start() function uses the character bytes given and
963 * find out the matching composition mappings if any and return the address
964 * to the composition mappings as explained in the do_composition().
967 find_composition_start(size_t uv, uchar_t *s, size_t sz)
980 } else if (sz == 2) {
983 } else if (sz == 3) {
987 } else if (sz == 4) {
994 * This is a fallback and should not happen if the function
995 * was called properly.
1000 b1 = u8_composition_b1_tbl[uv][b1];
1001 if (b1 == U8_TBL_ELEMENT_NOT_DEF)
1004 b2 = u8_composition_b2_tbl[uv][b1][b2];
1005 if (b2 == U8_TBL_ELEMENT_NOT_DEF)
1008 b3_tbl = u8_composition_b3_tbl[uv][b2][b3].tbl_id;
1009 if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF)
1012 if (b3_tbl >= U8_16BIT_TABLE_INDICATOR) {
1013 b3_tbl -= U8_16BIT_TABLE_INDICATOR;
1014 start_id = u8_composition_b4_16bit_tbl[uv][b3_tbl][b4];
1015 end_id = u8_composition_b4_16bit_tbl[uv][b3_tbl][b4 + 1];
1017 start_id = u8_composition_b4_tbl[uv][b3_tbl][b4];
1018 end_id = u8_composition_b4_tbl[uv][b3_tbl][b4 + 1];
1021 if (start_id >= end_id)
1024 b3_base = u8_composition_b3_tbl[uv][b2][b3].base;
1026 return ((uchar_t *)&(u8_composition_final_tbl[uv][b3_base + start_id]));
1030 * The blocked() function checks on the combining class values of previous
1031 * characters in this sequence and return whether it is blocked or not.
1034 blocked(uchar_t *comb_class, size_t last)
1036 uchar_t my_comb_class;
1039 my_comb_class = comb_class[last];
1040 for (i = 1; i < last; i++)
1041 if (comb_class[i] >= my_comb_class ||
1042 comb_class[i] == U8_COMBINING_CLASS_STARTER)
1049 * The do_composition() reads the character string pointed by 's' and
1050 * do necessary canonical composition and then copy over the result back to
1053 * The input argument 's' cannot contain more than 32 characters.
1056 do_composition(size_t uv, uchar_t *s, uchar_t *comb_class, uchar_t *start,
1057 uchar_t *disp, size_t last, uchar_t **os, uchar_t *oslast)
1059 uchar_t t[U8_STREAM_SAFE_TEXT_MAX + 1];
1060 uchar_t tc[U8_MB_CUR_MAX];
1061 uint8_t saved_marks[U8_MAX_CHARS_A_SEQ];
1062 size_t saved_marks_count;
1076 boolean_t match_not_found = B_TRUE;
1079 * This should never happen unless the callers are doing some strange
1080 * and unexpected things.
1082 * The "last" is the index pointing to the last character not last + 1.
1084 if (last >= U8_MAX_CHARS_A_SEQ)
1085 last = U8_UPPER_LIMIT_IN_A_SEQ;
1087 for (i = l = 0; i <= last; i++) {
1089 * The last or any non-Starters at the beginning, we don't
1090 * have any chance to do composition and so we just copy them
1091 * to the temporary buffer.
1093 if (i >= last || comb_class[i] != U8_COMBINING_CLASS_STARTER) {
1097 for (k = 0; k < size; k++)
1103 * If this could be a start of Hangul Jamos, then, we try to
1106 if (s[start[i]] == U8_HANGUL_JAMO_1ST_BYTE) {
1107 U8_PUT_3BYTES_INTO_UTF32(u1, s[start[i]],
1108 s[start[i] + 1], s[start[i] + 2]);
1109 U8_PUT_3BYTES_INTO_UTF32(u2, s[start[i] + 3],
1110 s[start[i] + 4], s[start[i] + 5]);
1112 if (U8_HANGUL_JAMO_L(u1) && U8_HANGUL_JAMO_V(u2)) {
1113 u1 -= U8_HANGUL_JAMO_L_FIRST;
1114 u2 -= U8_HANGUL_JAMO_V_FIRST;
1115 u1 = U8_HANGUL_SYL_FIRST +
1116 (u1 * U8_HANGUL_V_COUNT + u2) *
1121 U8_PUT_3BYTES_INTO_UTF32(u2,
1122 s[start[i]], s[start[i] + 1],
1125 if (U8_HANGUL_JAMO_T(u2)) {
1127 U8_HANGUL_JAMO_T_FIRST;
1132 U8_SAVE_HANGUL_AS_UTF8(t + l, 0, 1, 2, u1);
1140 * Let's then find out if this Starter has composition
1143 p = find_composition_start(uv, s + start[i], disp[i]);
1148 * We have a Starter with composition mapping and the next
1149 * character is a non-Starter. Let's try to find out if
1150 * we can do composition.
1156 saved_marks_count = 0;
1163 * The next for() loop compares the non-Starter pointed by
1164 * 'q' with the possible (joinable) characters pointed by 'p'.
1166 * The composition final table entry pointed by the 'p'
1167 * looks like the following:
1169 * +---+---+---+-...-+---+---+---+---+-...-+---+---+
1170 * | C | b0| b2| ... | bn| F | B0| B1| ... | Bm| F |
1171 * +---+---+---+-...-+---+---+---+---+-...-+---+---+
1173 * where C is the count byte indicating the number of
1174 * mapping pairs where each pair would be look like
1175 * (b0-bn F, B0-Bm F). The b0-bn are the bytes of the second
1176 * character of a canonical decomposition and the B0-Bm are
1177 * the bytes of a matching composite character. The F is
1178 * a filler byte after each character as the separator.
1181 match_not_found = B_TRUE;
1183 for (C = *p++; C > 0; C--) {
1184 for (k = 0; k < size; p++, k++)
1188 /* Have we found it? */
1189 if (k >= size && *p == U8_TBL_ELEMENT_FILLER) {
1190 match_not_found = B_FALSE;
1194 while (*++p != U8_TBL_ELEMENT_FILLER)
1200 /* We didn't find; skip to the next pair. */
1201 if (*p != U8_TBL_ELEMENT_FILLER)
1202 while (*++p != U8_TBL_ELEMENT_FILLER)
1204 while (*++p != U8_TBL_ELEMENT_FILLER)
1210 * If there was no match, we will need to save the combining
1211 * mark for later appending. After that, if the next one
1212 * is a non-Starter and not blocked, then, we try once
1213 * again to do composition with the next non-Starter.
1215 * If there was no match and this was a Starter, then,
1216 * this is a new start.
1218 * If there was a match and a composition done and we have
1219 * more to check on, then, we retrieve a new composition final
1220 * table entry for the composite and then try to do the
1221 * composition again.
1224 if (match_not_found) {
1225 if (comb_class[i] == U8_COMBINING_CLASS_STARTER) {
1230 saved_marks[saved_marks_count++] = i;
1235 if (blocked(comb_class, i + 1))
1236 saved_marks[saved_marks_count++] = ++i;
1242 goto TRY_THE_NEXT_MARK;
1244 } else if (i < last) {
1245 p = find_composition_start(uv, t + saved_l,
1249 goto TRY_THE_NEXT_MARK;
1254 * There is no more composition possible.
1256 * If there was no composition what so ever then we copy
1257 * over the original Starter and then append any non-Starters
1258 * remaining at the target string sequentially after that.
1262 p = s + start[saved_i];
1263 size = disp[saved_i];
1264 for (j = 0; j < size; j++)
1268 for (k = 0; k < saved_marks_count; k++) {
1269 p = s + start[saved_marks[k]];
1270 size = disp[saved_marks[k]];
1271 for (j = 0; j < size; j++)
1277 * If the last character is a Starter and if we have a character
1278 * (possibly another Starter) that can be turned into a composite,
1279 * we do so and we do so until there is no more of composition
1282 if (comb_class[last] == U8_COMBINING_CLASS_STARTER) {
1284 saved_l = l - disp[last];
1286 while (p < oslast) {
1287 size = u8_number_of_bytes[*p];
1288 if (size <= 1 || (p + size) > oslast)
1293 for (i = 0; i < size; i++)
1296 q = find_composition_start(uv, t + saved_l,
1303 match_not_found = B_TRUE;
1305 for (C = *q++; C > 0; C--) {
1306 for (k = 0; k < size; q++, k++)
1310 if (k >= size && *q == U8_TBL_ELEMENT_FILLER) {
1311 match_not_found = B_FALSE;
1315 while (*++q != U8_TBL_ELEMENT_FILLER) {
1317 * This is practically
1318 * impossible but we don't
1319 * want to take any chances.
1322 U8_STREAM_SAFE_TEXT_MAX) {
1332 if (*q != U8_TBL_ELEMENT_FILLER)
1333 while (*++q != U8_TBL_ELEMENT_FILLER)
1335 while (*++q != U8_TBL_ELEMENT_FILLER)
1340 if (match_not_found) {
1350 * Now we copy over the temporary string to the target string.
1351 * Since composition always reduces the number of characters or
1352 * the number of characters stay, we don't need to worry about
1353 * the buffer overflow here.
1355 for (i = 0; i < l; i++)
1363 * The collect_a_seq() function checks on the given string s, collect
1364 * a sequence of characters at u8s, and return the sequence. While it collects
1365 * a sequence, it also applies case conversion, canonical or compatibility
1366 * decomposition, canonical decomposition, or some or all of them and
1369 * The collected sequence cannot be bigger than 32 characters since if
1370 * it is having more than 31 characters, the sequence will be terminated
1371 * with a U+034F COMBINING GRAPHEME JOINER (CGJ) character and turned into
1372 * a Stream-Safe Text. The collected sequence is always terminated with
1373 * a null byte and the return value is the byte length of the sequence
1374 * including 0. The return value does not include the terminating
1378 collect_a_seq(size_t uv, uchar_t *u8s, uchar_t **source, uchar_t *slast,
1379 boolean_t is_it_toupper,
1380 boolean_t is_it_tolower,
1381 boolean_t canonical_decomposition,
1382 boolean_t compatibility_decomposition,
1383 boolean_t canonical_composition,
1384 int *errnum, u8_normalization_states_t *state)
1393 uchar_t comb_class[U8_MAX_CHARS_A_SEQ];
1394 uchar_t disp[U8_MAX_CHARS_A_SEQ];
1395 uchar_t start[U8_MAX_CHARS_A_SEQ];
1396 uchar_t u8t[U8_MB_CUR_MAX];
1397 uchar_t uts[U8_STREAM_SAFE_TEXT_MAX + 1];
1404 * Save the source string pointer which we will return a changed
1405 * pointer if we do processing.
1410 * The following is a fallback for just in case callers are not
1411 * checking the string boundaries before the calling.
1420 * As the first thing, let's collect a character and do case
1421 * conversion if necessary.
1424 sz = u8_number_of_bytes[*s];
1439 u8s[0] = U8_ASCII_TOUPPER(*s);
1440 else if (is_it_tolower)
1441 u8s[0] = U8_ASCII_TOLOWER(*s);
1446 } else if ((s + sz) > slast) {
1449 for (i = 0; s < slast; )
1457 if (is_it_toupper || is_it_tolower) {
1458 i = do_case_conv(uv, u8s, s, sz, is_it_toupper);
1462 for (i = 0; i < sz; )
1469 * And then canonical/compatibility decomposition followed by
1470 * an optional canonical composition. Please be noted that
1471 * canonical composition is done only when a decomposition is
1474 if (canonical_decomposition || compatibility_decomposition) {
1476 *state = U8_STATE_START;
1486 saved_sz = do_decomp(uv, u8s, u8s, sz,
1487 canonical_decomposition, state);
1491 for (i = 0; i < saved_sz; ) {
1492 sz = u8_number_of_bytes[u8s[i]];
1494 comb_class[last] = combining_class(uv,
1504 * Decomposition yields various Hangul related
1505 * states but not on combining marks. We need to
1506 * find out at here by checking on the last
1509 if (*state == U8_STATE_START) {
1510 if (comb_class[last - 1])
1511 *state = U8_STATE_COMBINING_MARK;
1518 sz = u8_number_of_bytes[*s];
1521 * If this is an illegal character, an incomplete
1522 * character, or an 7-bit ASCII Starter character,
1523 * then we have collected a sequence; break and let
1524 * the next call deal with the two cases.
1526 * Note that this is okay only if you are using this
1527 * function with a fixed length string, not on
1528 * a buffer with multiple calls of one chunk at a time.
1532 } else if ((s + sz) > slast) {
1536 * If the previous character was a Hangul Jamo
1537 * and this character is a Hangul Jamo that
1538 * can be conjoined, we collect the Jamo.
1540 if (*s == U8_HANGUL_JAMO_1ST_BYTE) {
1541 U8_PUT_3BYTES_INTO_UTF32(u1,
1542 *s, *(s + 1), *(s + 2));
1544 if (U8_HANGUL_COMPOSABLE_L_V(*state,
1547 *state = U8_STATE_HANGUL_LV;
1548 goto COLLECT_A_HANGUL;
1551 if (U8_HANGUL_COMPOSABLE_LV_T(*state,
1554 *state = U8_STATE_HANGUL_LVT;
1555 goto COLLECT_A_HANGUL;
1560 * Regardless of whatever it was, if this is
1561 * a Starter, we don't collect the character
1562 * since that's a new start and we will deal
1563 * with it at the next time.
1565 i = combining_class(uv, s, sz);
1566 if (i == U8_COMBINING_CLASS_STARTER)
1570 * We know the current character is a combining
1571 * mark. If the previous character wasn't
1572 * a Starter (not Hangul) or a combining mark,
1573 * then, we don't collect this combining mark.
1575 if (*state != U8_STATE_START &&
1576 *state != U8_STATE_COMBINING_MARK)
1579 *state = U8_STATE_COMBINING_MARK;
1582 * If we collected a Starter and combining
1583 * marks up to 30, i.e., total 31 characters,
1584 * then, we terminate this degenerately long
1585 * combining sequence with a U+034F COMBINING
1586 * GRAPHEME JOINER (CGJ) which is 0xCD 0x8F in
1587 * UTF-8 and turn this into a Stream-Safe
1588 * Text. This will be extremely rare but
1591 * The following will also guarantee that
1592 * we are not writing more than 32 characters
1593 * plus a NULL at u8s[].
1595 if (last >= U8_UPPER_LIMIT_IN_A_SEQ) {
1597 *state = U8_STATE_START;
1598 comb_class[last] = 0;
1599 start[last] = saved_sz;
1603 u8s[saved_sz++] = 0xCD;
1604 u8s[saved_sz++] = 0x8F;
1610 * Some combining marks also do decompose into
1611 * another combining mark or marks.
1613 if (*state == U8_STATE_COMBINING_MARK) {
1616 i = do_decomp(uv, uts, s, sz,
1617 canonical_decomposition, state);
1618 for (j = 0; j < i; ) {
1619 sz = u8_number_of_bytes[uts[j]];
1624 start[last] = saved_sz + j;
1629 U8_UPPER_LIMIT_IN_A_SEQ) {
1631 goto TURN_STREAM_SAFE;
1636 *state = U8_STATE_COMBINING_MARK;
1640 for (i = 0; i < sz; i++)
1641 u8s[saved_sz++] = uts[i];
1643 comb_class[last] = i;
1644 start[last] = saved_sz;
1648 for (i = 0; i < sz; i++)
1649 u8s[saved_sz++] = *s++;
1653 * If this is U+0345 COMBINING GREEK
1654 * YPOGEGRAMMENI (0xCD 0x85 in UTF-8), a.k.a.,
1655 * iota subscript, and need to be converted to
1656 * uppercase letter, convert it to U+0399 GREEK
1657 * CAPITAL LETTER IOTA (0xCE 0x99 in UTF-8),
1658 * i.e., convert to capital adscript form as
1659 * specified in the Unicode standard.
1661 * This is the only special case of (ambiguous)
1662 * case conversion at combining marks and
1663 * probably the standard will never have
1664 * anything similar like this in future.
1666 if (is_it_toupper && sz >= 2 &&
1667 u8s[saved_sz - 2] == 0xCD &&
1668 u8s[saved_sz - 1] == 0x85) {
1669 u8s[saved_sz - 2] = 0xCE;
1670 u8s[saved_sz - 1] = 0x99;
1676 * Let's try to ensure a canonical ordering for the collected
1677 * combining marks. We do this only if we have collected
1678 * at least one more non-Starter. (The decomposition mapping
1679 * data tables have fully (and recursively) expanded and
1680 * canonically ordered decompositions.)
1682 * The U8_SWAP_COMB_MARKS() convenience macro has some
1683 * assumptions and we are meeting the assumptions.
1686 if (last >= saved_last) {
1687 for (i = 0; i < last; i++)
1688 for (j = last; j > i; j--)
1689 if (comb_class[j] &&
1690 comb_class[j - 1] > comb_class[j]) {
1691 U8_SWAP_COMB_MARKS(j - 1, j);
1697 if (! canonical_composition) {
1698 u8s[saved_sz] = '\0';
1703 * Now do the canonical composition. Note that we do this
1704 * only after a canonical or compatibility decomposition to
1705 * finish up NFC or NFKC.
1707 sz = do_composition(uv, u8s, comb_class, start, disp, last,
1713 return ((size_t)sz);
1717 * The do_norm_compare() function does string comparion based on Unicode
1718 * simple case mappings and Unicode Normalization definitions.
1720 * It does so by collecting a sequence of character at a time and comparing
1721 * the collected sequences from the strings.
1723 * The meanings on the return values are the same as the usual strcmp().
1726 do_norm_compare(size_t uv, uchar_t *s1, uchar_t *s2, size_t n1, size_t n2,
1727 int flag, int *errnum)
1732 uchar_t u8s1[U8_STREAM_SAFE_TEXT_MAX + 1];
1733 uchar_t u8s2[U8_STREAM_SAFE_TEXT_MAX + 1];
1736 boolean_t is_it_toupper;
1737 boolean_t is_it_tolower;
1738 boolean_t canonical_decomposition;
1739 boolean_t compatibility_decomposition;
1740 boolean_t canonical_composition;
1741 u8_normalization_states_t state;
1746 is_it_toupper = flag & U8_TEXTPREP_TOUPPER;
1747 is_it_tolower = flag & U8_TEXTPREP_TOLOWER;
1748 canonical_decomposition = flag & U8_CANON_DECOMP;
1749 compatibility_decomposition = flag & U8_COMPAT_DECOMP;
1750 canonical_composition = flag & U8_CANON_COMP;
1752 while (s1 < s1last && s2 < s2last) {
1754 * If the current character is a 7-bit ASCII and the last
1755 * character, or, if the current character and the next
1756 * character are both some 7-bit ASCII characters then
1757 * we treat the current character as a sequence.
1759 * In any other cases, we need to call collect_a_seq().
1762 if (U8_ISASCII(*s1) && ((s1 + 1) >= s1last ||
1763 ((s1 + 1) < s1last && U8_ISASCII(*(s1 + 1))))) {
1765 u8s1[0] = U8_ASCII_TOUPPER(*s1);
1766 else if (is_it_tolower)
1767 u8s1[0] = U8_ASCII_TOLOWER(*s1);
1774 state = U8_STATE_START;
1775 sz1 = collect_a_seq(uv, u8s1, &s1, s1last,
1776 is_it_toupper, is_it_tolower,
1777 canonical_decomposition,
1778 compatibility_decomposition,
1779 canonical_composition, errnum, &state);
1782 if (U8_ISASCII(*s2) && ((s2 + 1) >= s2last ||
1783 ((s2 + 1) < s2last && U8_ISASCII(*(s2 + 1))))) {
1785 u8s2[0] = U8_ASCII_TOUPPER(*s2);
1786 else if (is_it_tolower)
1787 u8s2[0] = U8_ASCII_TOLOWER(*s2);
1794 state = U8_STATE_START;
1795 sz2 = collect_a_seq(uv, u8s2, &s2, s2last,
1796 is_it_toupper, is_it_tolower,
1797 canonical_decomposition,
1798 compatibility_decomposition,
1799 canonical_composition, errnum, &state);
1803 * Now compare the two characters. If they are the same,
1804 * we move on to the next character sequences.
1806 if (sz1 == 1 && sz2 == 1) {
1812 result = strcmp((const char *)u8s1, (const char *)u8s2);
1819 * We compared until the end of either or both strings.
1821 * If we reached to or went over the ends for the both, that means
1822 * they are the same.
1824 * If we reached only one end, that means the other string has
1825 * something which then can be used to determine the return value.
1836 * The u8_strcmp() function compares two UTF-8 strings quite similar to
1837 * the strcmp(). For the comparison, however, Unicode Normalization specific
1838 * equivalency and Unicode simple case conversion mappings based equivalency
1839 * can be requested and checked against.
1842 u8_strcmp(const char *s1, const char *s2, size_t n, int flag, size_t uv,
1852 * Check on the requested Unicode version, case conversion, and
1853 * normalization flag values.
1856 if (uv > U8_UNICODE_LATEST) {
1858 uv = U8_UNICODE_LATEST;
1862 flag = U8_STRCMP_CS;
1864 f = flag & (U8_STRCMP_CS | U8_STRCMP_CI_UPPER |
1865 U8_STRCMP_CI_LOWER);
1867 flag |= U8_STRCMP_CS;
1868 } else if (f != U8_STRCMP_CS && f != U8_STRCMP_CI_UPPER &&
1869 f != U8_STRCMP_CI_LOWER) {
1871 flag = U8_STRCMP_CS;
1874 f = flag & (U8_CANON_DECOMP | U8_COMPAT_DECOMP | U8_CANON_COMP);
1875 if (f && f != U8_STRCMP_NFD && f != U8_STRCMP_NFC &&
1876 f != U8_STRCMP_NFKD && f != U8_STRCMP_NFKC) {
1878 flag = U8_STRCMP_CS;
1882 if (flag == U8_STRCMP_CS) {
1883 return (n == 0 ? strcmp(s1, s2) : strncmp(s1, s2, n));
1896 * Simple case conversion can be done much faster and so we do
1897 * them separately here.
1899 if (flag == U8_STRCMP_CI_UPPER) {
1900 return (do_case_compare(uv, (uchar_t *)s1, (uchar_t *)s2,
1901 n1, n2, B_TRUE, errnum));
1902 } else if (flag == U8_STRCMP_CI_LOWER) {
1903 return (do_case_compare(uv, (uchar_t *)s1, (uchar_t *)s2,
1904 n1, n2, B_FALSE, errnum));
1907 return (do_norm_compare(uv, (uchar_t *)s1, (uchar_t *)s2, n1, n2,
1912 u8_textprep_str(char *inarray, size_t *inlen, char *outarray, size_t *outlen,
1913 int flag, size_t unicode_version, int *errnum)
1921 boolean_t do_not_ignore_null;
1922 boolean_t do_not_ignore_invalid;
1923 boolean_t is_it_toupper;
1924 boolean_t is_it_tolower;
1925 boolean_t canonical_decomposition;
1926 boolean_t compatibility_decomposition;
1927 boolean_t canonical_composition;
1931 uchar_t u8s[U8_STREAM_SAFE_TEXT_MAX + 1];
1932 u8_normalization_states_t state;
1934 if (unicode_version > U8_UNICODE_LATEST) {
1936 return ((size_t)-1);
1939 f = flag & (U8_TEXTPREP_TOUPPER | U8_TEXTPREP_TOLOWER);
1940 if (f == (U8_TEXTPREP_TOUPPER | U8_TEXTPREP_TOLOWER)) {
1942 return ((size_t)-1);
1945 f = flag & (U8_CANON_DECOMP | U8_COMPAT_DECOMP | U8_CANON_COMP);
1946 if (f && f != U8_TEXTPREP_NFD && f != U8_TEXTPREP_NFC &&
1947 f != U8_TEXTPREP_NFKD && f != U8_TEXTPREP_NFKC) {
1949 return ((size_t)-1);
1952 if (inarray == NULL || *inlen == 0)
1955 if (outarray == NULL) {
1957 return ((size_t)-1);
1960 ib = (uchar_t *)inarray;
1961 ob = (uchar_t *)outarray;
1962 ibtail = ib + *inlen;
1963 obtail = ob + *outlen;
1965 do_not_ignore_null = !(flag & U8_TEXTPREP_IGNORE_NULL);
1966 do_not_ignore_invalid = !(flag & U8_TEXTPREP_IGNORE_INVALID);
1967 is_it_toupper = flag & U8_TEXTPREP_TOUPPER;
1968 is_it_tolower = flag & U8_TEXTPREP_TOLOWER;
1973 * If we don't have a normalization flag set, we do the simple case
1974 * conversion based text preparation separately below. Text
1975 * preparation involving Normalization will be done in the false task
1976 * block, again, separately since it will take much more time and
1977 * resource than doing simple case conversions.
1980 while (ib < ibtail) {
1981 if (*ib == '\0' && do_not_ignore_null)
1984 sz = u8_number_of_bytes[*ib];
1987 if (do_not_ignore_invalid) {
1989 ret_val = (size_t)-1;
2000 ret_val = (size_t)-1;
2005 *ob = U8_ASCII_TOUPPER(*ib);
2006 else if (is_it_tolower)
2007 *ob = U8_ASCII_TOLOWER(*ib);
2012 } else if ((ib + sz) > ibtail) {
2013 if (do_not_ignore_invalid) {
2015 ret_val = (size_t)-1;
2019 if ((obtail - ob) < (ibtail - ib)) {
2021 ret_val = (size_t)-1;
2026 * We treat the remaining incomplete character
2027 * bytes as a character.
2034 if (is_it_toupper || is_it_tolower) {
2035 i = do_case_conv(unicode_version, u8s,
2036 ib, sz, is_it_toupper);
2038 if ((obtail - ob) < i) {
2040 ret_val = (size_t)-1;
2046 for (sz = 0; sz < i; sz++)
2049 if ((obtail - ob) < sz) {
2051 ret_val = (size_t)-1;
2055 for (i = 0; i < sz; i++)
2061 canonical_decomposition = flag & U8_CANON_DECOMP;
2062 compatibility_decomposition = flag & U8_COMPAT_DECOMP;
2063 canonical_composition = flag & U8_CANON_COMP;
2065 while (ib < ibtail) {
2066 if (*ib == '\0' && do_not_ignore_null)
2070 * If the current character is a 7-bit ASCII
2071 * character and it is the last character, or,
2072 * if the current character is a 7-bit ASCII
2073 * character and the next character is also a 7-bit
2074 * ASCII character, then, we copy over this
2075 * character without going through collect_a_seq().
2077 * In any other cases, we need to look further with
2078 * the collect_a_seq() function.
2080 if (U8_ISASCII(*ib) && ((ib + 1) >= ibtail ||
2081 ((ib + 1) < ibtail && U8_ISASCII(*(ib + 1))))) {
2084 ret_val = (size_t)-1;
2089 *ob = U8_ASCII_TOUPPER(*ib);
2090 else if (is_it_tolower)
2091 *ob = U8_ASCII_TOLOWER(*ib);
2098 state = U8_STATE_START;
2100 j = collect_a_seq(unicode_version, u8s,
2104 canonical_decomposition,
2105 compatibility_decomposition,
2106 canonical_composition,
2109 if (*errnum && do_not_ignore_invalid) {
2110 ret_val = (size_t)-1;
2114 if ((obtail - ob) < j) {
2116 ret_val = (size_t)-1;
2120 for (i = 0; i < j; i++)
2126 *inlen = ibtail - ib;
2127 *outlen = obtail - ob;