2 * Copyright 2018 Nexenta Systems, Inc.
3 * Copyright 2012 Garrett D'Amore <garrett@damore.org> All rights reserved.
4 * Copyright 2015 John Marino <draco@marino.st>
6 * This source code is derived from the illumos localedef command, and
7 * provided under BSD-style license terms by Nexenta Systems, Inc.
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
33 * LC_CTYPE database generation routines for localedef.
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD$");
44 #include <sys/types.h>
47 #include "localedef.h"
50 /* Always include the defines for the target: */
51 #define _DONT_USE_CTYPE_INLINE_ /* Avoid dependencies on runetype.h */
56 /* Needed for bootstrapping, _CTYPE_N */
58 #define _CTYPE_N 0x00400000L
61 #define _ISUPPER _CTYPE_U
62 #define _ISLOWER _CTYPE_L
63 #define _ISDIGIT _CTYPE_D
64 #define _ISXDIGIT _CTYPE_X
65 #define _ISSPACE _CTYPE_S
66 #define _ISBLANK _CTYPE_B
67 #define _ISALPHA _CTYPE_A
68 #define _ISPUNCT _CTYPE_P
69 #define _ISGRAPH _CTYPE_G
70 #define _ISPRINT _CTYPE_R
71 #define _ISCNTRL _CTYPE_C
78 static wchar_t last_ctype;
79 static int ctype_compare(const void *n1, const void *n2);
81 typedef struct ctype_node {
86 RB_ENTRY(ctype_node) entry;
89 static RB_HEAD(ctypes, ctype_node) ctypes;
90 RB_GENERATE_STATIC(ctypes, ctype_node, entry, ctype_compare);
93 ctype_compare(const void *n1, const void *n2)
95 const ctype_node_t *c1 = n1;
96 const ctype_node_t *c2 = n2;
98 return (c1->wc < c2->wc ? -1 : c1->wc > c2->wc ? 1 : 0);
109 add_ctype_impl(ctype_node_t *ctn)
113 ctn->ctype |= (_ISUPPER | _ISALPHA | _ISGRAPH | _ISPRINT);
116 ctn->ctype |= (_ISLOWER | _ISALPHA | _ISGRAPH | _ISPRINT);
119 ctn->ctype |= (_ISALPHA | _ISGRAPH | _ISPRINT);
122 ctn->ctype |= (_ISDIGIT | _ISGRAPH | _ISPRINT | _ISXDIGIT | _E4);
126 * This can be troublesome as <form-feed>, <newline>,
127 * <carriage-return>, <tab>, and <vertical-tab> are defined both
128 * as space and cntrl, and POSIX doesn't allow cntrl/print
129 * combination. We will take care of this in dump_ctype().
131 ctn->ctype |= (_ISSPACE | _ISPRINT);
134 ctn->ctype |= _ISCNTRL;
137 ctn->ctype |= (_ISGRAPH | _ISPRINT);
140 ctn->ctype |= _ISPRINT;
143 ctn->ctype |= (_ISPUNCT | _ISGRAPH | _ISPRINT);
146 ctn->ctype |= (_ISXDIGIT | _ISPRINT);
149 ctn->ctype |= (_ISBLANK | _ISSPACE);
152 ctn->ctype |= (_E1 | _ISPRINT | _ISGRAPH);
155 ctn->ctype |= (_E2 | _ISPRINT | _ISGRAPH);
158 ctn->ctype |= (_E3 | _ISPRINT | _ISGRAPH);
161 ctn->ctype |= (_E4 | _ISPRINT | _ISGRAPH);
164 ctn->ctype |= (_E5 | _ISPRINT | _ISGRAPH);
168 * We can't do anything with this. The character
169 * should already be specified as a digit or alpha.
173 errf("not a valid character class");
177 static ctype_node_t *
178 get_ctype(wchar_t wc)
184 if ((ctn = RB_FIND(ctypes, &ctypes, &srch)) == NULL) {
185 if ((ctn = calloc(1, sizeof (*ctn))) == NULL) {
186 errf("out of memory");
191 RB_INSERT(ctypes, &ctypes, ctn);
201 if ((ctn = get_ctype(val)) == NULL) {
206 last_ctype = ctn->wc;
210 add_ctype_range(wchar_t end)
215 if (end < last_ctype) {
216 errf("malformed character range (%u ... %u))",
220 for (cur = last_ctype + 1; cur <= end; cur++) {
221 if ((ctn = get_ctype(cur)) == NULL) {
232 * A word about widths: if the width mask is specified, then libc
233 * unconditionally honors it. Otherwise, it assumes printable
234 * characters have width 1, and non-printable characters have width
235 * -1 (except for NULL which is special with width 0). Hence, we have
236 * no need to inject defaults here -- the "default" unset value of 0
237 * indicates that libc should use its own logic in wcwidth as described.
240 add_width(int wc, int width)
244 if ((ctn = get_ctype(wc)) == NULL) {
248 ctn->ctype &= ~(_CTYPE_SWM);
251 ctn->ctype |= _CTYPE_SW0;
254 ctn->ctype |= _CTYPE_SW1;
257 ctn->ctype |= _CTYPE_SW2;
260 ctn->ctype |= _CTYPE_SW3;
266 add_width_range(int start, int end, int width)
268 for (; start <= end; start++) {
269 add_width(start, width);
274 add_caseconv(int val, int wc)
278 ctn = get_ctype(val);
302 ctype_node_t *ctn, *last_ct, *last_lo, *last_up;
303 _FileRuneEntry *ct = NULL;
304 _FileRuneEntry *lo = NULL;
305 _FileRuneEntry *up = NULL;
307 uint32_t runetype_ext_nranges;
308 uint32_t maplower_ext_nranges;
309 uint32_t mapupper_ext_nranges;
311 (void) memset(&rl, 0, sizeof (rl));
312 runetype_ext_nranges = 0;
314 maplower_ext_nranges = 0;
316 mapupper_ext_nranges = 0;
319 if ((f = open_category()) == NULL)
322 (void) memcpy(rl.magic, _FILE_RUNE_MAGIC_1, 8);
323 (void) strlcpy(rl.encoding, get_wide_encoding(), sizeof (rl.encoding));
326 * Initialize the identity map.
328 for (wc = 0; (unsigned)wc < _CACHED_RUNES; wc++) {
329 rl.maplower[wc] = htote(wc);
330 rl.mapupper[wc] = htote(wc);
333 RB_FOREACH(ctn, ctypes, &ctypes) {
339 * POSIX requires certain portable characters have
340 * certain types. Add them if they are missing.
342 if ((wc >= 1) && (wc <= 127)) {
343 if ((wc >= 'A') && (wc <= 'Z'))
344 ctn->ctype |= _ISUPPER;
345 if ((wc >= 'a') && (wc <= 'z'))
346 ctn->ctype |= _ISLOWER;
347 if ((wc >= '0') && (wc <= '9'))
348 ctn->ctype |= _ISDIGIT;
350 ctn->ctype |= _ISPRINT;
351 if (strchr(" \f\n\r\t\v", (char)wc) != NULL)
352 ctn->ctype |= _ISSPACE;
353 if (strchr("0123456789ABCDEFabcdef", (char)wc) != NULL)
354 ctn->ctype |= _ISXDIGIT;
355 if (strchr(" \t", (char)wc))
356 ctn->ctype |= _ISBLANK;
359 * Technically these settings are only
360 * required for the C locale. However, it
361 * turns out that because of the historical
362 * version of isprint(), we need them for all
363 * locales as well. Note that these are not
364 * necessarily valid punctation characters in
365 * the current language, but ispunct() needs
366 * to return TRUE for them.
368 if (strchr("!\"'#$%&()*+,-./:;<=>?@[\\]^_`{|}~",
370 ctn->ctype |= _ISPUNCT;
374 * POSIX also requires that certain types imply
375 * others. Add any inferred types here.
377 if (ctn->ctype & (_ISUPPER |_ISLOWER))
378 ctn->ctype |= _ISALPHA;
379 if (ctn->ctype & _ISDIGIT)
380 ctn->ctype |= _ISXDIGIT;
381 if (ctn->ctype & _ISBLANK)
382 ctn->ctype |= _ISSPACE;
383 if (ctn->ctype & (_ISALPHA|_ISDIGIT|_ISXDIGIT))
384 ctn->ctype |= _ISGRAPH;
385 if (ctn->ctype & _ISGRAPH)
386 ctn->ctype |= _ISPRINT;
389 * POSIX requires that certain combinations are invalid.
390 * Try fixing the cases we know about (see add_ctype_impl()).
392 if ((ctn->ctype & (_ISSPACE|_ISCNTRL)) == (_ISSPACE|_ISCNTRL))
393 ctn->ctype &= ~_ISPRINT;
396 * Finally, don't flag remaining cases as a fatal error,
397 * and just warn about them.
399 if ((ctn->ctype & _ISALPHA) &&
400 (ctn->ctype & (_ISPUNCT|_ISDIGIT)))
402 if ((ctn->ctype & _ISPUNCT) &&
403 (ctn->ctype & (_ISDIGIT|_ISALPHA|_ISXDIGIT)))
405 if ((ctn->ctype & _ISSPACE) && (ctn->ctype & _ISGRAPH))
407 if ((ctn->ctype & _ISCNTRL) && (ctn->ctype & _ISPRINT))
409 if ((wc == ' ') && (ctn->ctype & (_ISPUNCT|_ISGRAPH)))
413 warn("conflicting classes for character 0x%x (%x)",
417 * Handle the lower 256 characters using the simple
418 * optimization. Note that if we have not defined the
419 * upper/lower case, then we identity map it.
421 if ((unsigned)wc < _CACHED_RUNES) {
422 rl.runetype[wc] = htote(ctn->ctype);
424 rl.maplower[wc] = htote(ctn->tolower);
426 rl.mapupper[wc] = htote(ctn->toupper);
430 if ((last_ct != NULL) && (last_ct->ctype == ctn->ctype) &&
431 (last_ct->wc + 1 == wc)) {
432 ct[runetype_ext_nranges - 1].max = htote(wc);
434 runetype_ext_nranges++;
435 ct = realloc(ct, sizeof (*ct) * runetype_ext_nranges);
436 ct[runetype_ext_nranges - 1].min = htote(wc);
437 ct[runetype_ext_nranges - 1].max = htote(wc);
438 ct[runetype_ext_nranges - 1].map =
442 if (ctn->tolower == 0) {
444 } else if ((last_lo != NULL) &&
445 (last_lo->tolower + 1 == ctn->tolower)) {
446 lo[maplower_ext_nranges - 1].max = htote(wc);
449 maplower_ext_nranges++;
450 lo = realloc(lo, sizeof (*lo) * maplower_ext_nranges);
451 lo[maplower_ext_nranges - 1].min = htote(wc);
452 lo[maplower_ext_nranges - 1].max = htote(wc);
453 lo[maplower_ext_nranges - 1].map =
458 if (ctn->toupper == 0) {
460 } else if ((last_up != NULL) &&
461 (last_up->toupper + 1 == ctn->toupper)) {
462 up[mapupper_ext_nranges-1].max = htote(wc);
465 mapupper_ext_nranges++;
466 up = realloc(up, sizeof (*up) * mapupper_ext_nranges);
467 up[mapupper_ext_nranges - 1].min = htote(wc);
468 up[mapupper_ext_nranges - 1].max = htote(wc);
469 up[mapupper_ext_nranges - 1].map =
475 rl.runetype_ext_nranges = htote(runetype_ext_nranges);
476 rl.maplower_ext_nranges = htote(maplower_ext_nranges);
477 rl.mapupper_ext_nranges = htote(mapupper_ext_nranges);
478 if ((wr_category(&rl, sizeof (rl), f) < 0) ||
479 (wr_category(ct, sizeof (*ct) * runetype_ext_nranges, f) < 0) ||
480 (wr_category(lo, sizeof (*lo) * maplower_ext_nranges, f) < 0) ||
481 (wr_category(up, sizeof (*up) * mapupper_ext_nranges, f) < 0)) {