2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright 2014 Garrett D'Amore <garrett@damore.org>
5 * Copyright 2010 Nexenta Systems, Inc. All rights reserved.
6 * Copyright (c) 1995 Alex Tatmanjants <alex@elvisti.kiev.ua>
7 * at Electronni Visti IA, Kiev, Ukraine.
10 * Copyright (c) 2011 The FreeBSD Foundation
12 * Portions of this software were developed by David Chisnall
13 * under sponsorship from the FreeBSD Foundation.
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * Adapted to xlocale by John Marino <draco@marino.st>
39 #include <sys/cdefs.h>
40 __FBSDID("$FreeBSD$");
42 #include "namespace.h"
44 #include <sys/types.h>
56 #include "un-namespace.h"
59 #include "setlocale.h"
61 #include "libc_private.h"
63 struct xlocale_collate __xlocale_global_collate = {
64 {{0}, "C"}, 1, 0, 0, 0
67 struct xlocale_collate __xlocale_C_collate = {
68 {{0}, "C"}, 1, 0, 0, 0
71 struct xlocale_collate __xlocale_POSIX_collate = {
72 {{0}, "POSIX"}, 1, 0, 0, 0
75 struct xlocale_collate __xlocale_CUTF8_collate = {
76 {{0}, "C.UTF-8"}, 1, 0, 0, 0
80 __collate_load_tables_l(const char *encoding, struct xlocale_collate *table);
83 destruct_collate(void *t)
85 struct xlocale_collate *table = t;
86 if (table->map && (table->maplen > 0)) {
87 (void) munmap(table->map, table->maplen);
93 __collate_load(const char *encoding, __unused locale_t unused)
95 if (strcmp(encoding, "C") == 0)
96 return (&__xlocale_C_collate);
97 else if (strcmp(encoding, "POSIX") == 0)
98 return (&__xlocale_POSIX_collate);
99 else if (strcmp(encoding, "C.UTF-8") == 0)
100 return (&__xlocale_CUTF8_collate);
102 struct xlocale_collate *table = calloc(sizeof(struct xlocale_collate),
106 table->header.header.destructor = destruct_collate;
109 * FIXME: Make sure that _LDP_CACHE is never returned. We
110 * should be doing the caching outside of this section.
112 if (__collate_load_tables_l(encoding, table) != _LDP_LOADED) {
113 xlocale_release(table);
120 * Load the collation tables for the specified encoding into the global table.
123 __collate_load_tables(const char *encoding)
126 return (__collate_load_tables_l(encoding, &__xlocale_global_collate));
130 __collate_load_tables_l(const char *encoding, struct xlocale_collate *table)
136 collate_info_t *info;
140 table->__collate_load_error = 1;
142 /* 'encoding' must be already checked. */
143 if (strcmp(encoding, "C") == 0 || strcmp(encoding, "POSIX") == 0 ||
144 strncmp(encoding, "C.", 2) == 0) {
148 if (asprintf(&buf, "%s/%s/LC_COLLATE", _PathLocale, encoding) == -1)
151 if ((fd = _open(buf, O_RDONLY | O_CLOEXEC)) < 0) {
156 if (_fstat(fd, &sbuf) < 0) {
160 if (sbuf.st_size < (COLLATE_FMT_VERSION_LEN +
161 XLOCALE_DEF_VERSION_LEN +
167 map = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
169 if ((TMP = map) == MAP_FAILED) {
173 if (strncmp(TMP, COLLATE_FMT_VERSION, COLLATE_FMT_VERSION_LEN) != 0) {
174 (void) munmap(map, sbuf.st_size);
178 TMP += COLLATE_FMT_VERSION_LEN;
179 strlcat(table->header.version, TMP, sizeof (table->header.version));
180 TMP += XLOCALE_DEF_VERSION_LEN;
183 TMP += sizeof (*info);
185 if ((info->directive_count < 1) ||
186 (info->directive_count >= COLL_WEIGHTS_MAX) ||
187 ((chains = info->chain_count) < 0)) {
188 (void) munmap(map, sbuf.st_size);
193 i = (sizeof (collate_char_t) * (UCHAR_MAX + 1)) +
194 (sizeof (collate_chain_t) * chains) +
195 (sizeof (collate_large_t) * info->large_count);
196 for (z = 0; z < info->directive_count; z++) {
197 i += sizeof (collate_subst_t) * info->subst_count[z];
199 if (i != (sbuf.st_size - (TMP - map))) {
200 (void) munmap(map, sbuf.st_size);
205 if (table->map && (table->maplen > 0)) {
206 (void) munmap(table->map, table->maplen);
209 table->maplen = sbuf.st_size;
211 table->char_pri_table = (void *)TMP;
212 TMP += sizeof (collate_char_t) * (UCHAR_MAX + 1);
214 for (z = 0; z < info->directive_count; z++) {
215 if (info->subst_count[z] > 0) {
216 table->subst_table[z] = (void *)TMP;
217 TMP += info->subst_count[z] * sizeof (collate_subst_t);
219 table->subst_table[z] = NULL;
224 table->chain_pri_table = (void *)TMP;
225 TMP += chains * sizeof (collate_chain_t);
227 table->chain_pri_table = NULL;
228 if (info->large_count > 0)
229 table->large_pri_table = (void *)TMP;
231 table->large_pri_table = NULL;
233 table->__collate_load_error = 0;
234 return (_LDP_LOADED);
237 static const int32_t *
238 substsearch(struct xlocale_collate *table, const wchar_t key, int pass)
240 const collate_subst_t *p;
241 int n = table->info->subst_count[pass];
246 if (pass >= table->info->directive_count)
249 if (!(key & COLLATE_SUBST_PRIORITY))
252 p = table->subst_table[pass] + (key & ~COLLATE_SUBST_PRIORITY);
253 assert(p->key == key);
258 static collate_chain_t *
259 chainsearch(struct xlocale_collate *table, const wchar_t *key, int *len)
262 int high = table->info->chain_count - 1;
265 collate_chain_t *tab = table->chain_pri_table;
270 while (low <= high) {
271 next = (low + high) / 2;
273 compar = *key - *p->str;
275 l = wcsnlen(p->str, COLLATE_STR_LEN);
276 compar = wcsncmp(key, p->str, l);
290 static collate_large_t *
291 largesearch(struct xlocale_collate *table, const wchar_t key)
294 int high = table->info->large_count - 1;
297 collate_large_t *tab = table->large_pri_table;
302 while (low <= high) {
303 next = (low + high) / 2;
305 compar = key - p->val;
317 _collate_lookup(struct xlocale_collate *table, const wchar_t *t, int *len,
318 int *pri, int which, const int **state)
321 collate_large_t *match;
326 * If this is the "last" pass for the UNDEFINED, then
327 * we just return the priority itself.
329 if (which >= table->info->directive_count) {
337 * If we have remaining substitution data from a previous
338 * call, consume it first.
340 if ((sptr = *state) != NULL) {
343 if ((sptr == *state) || (sptr == NULL))
351 /* No active substitutions */
355 * Check for composites such as diphthongs that collate as a
356 * single element (aka chains or collating-elements).
358 if (((p2 = chainsearch(table, t, &l)) != NULL) &&
359 ((p = p2->pri[which]) >= 0)) {
364 } else if (*t <= UCHAR_MAX) {
367 * Character is a small (8-bit) character.
368 * We just look these up directly for speed.
370 *pri = table->char_pri_table[*t].pri[which];
372 } else if ((table->info->large_count > 0) &&
373 ((match = largesearch(table, *t)) != NULL)) {
376 * Character was found in the extended table.
378 *pri = match->pri.pri[which];
382 * Character lacks a specific definition.
384 if (table->info->directive[which] & DIRECTIVE_UNDEFINED) {
385 /* Mask off sign bit to prevent ordering confusion. */
386 *pri = (*t & COLLATE_MAX_PRIORITY);
388 *pri = table->info->undef_pri[which];
390 /* No substitutions for undefined characters! */
395 * Try substituting (expanding) the character. We are
396 * currently doing this *after* the chain compression. I
397 * think it should not matter, but this way might be slightly
400 * We do this after the priority search, as this will help us
401 * to identify a single key value. In order for this to work,
402 * its important that the priority assigned to a given element
403 * to be substituted be unique for that level. The localedef
404 * code ensures this for us.
406 if ((sptr = substsearch(table, *pri, which)) != NULL) {
407 if ((*pri = *sptr) > 0) {
409 *state = *sptr ? sptr : NULL;
416 * This is the meaty part of wcsxfrm & strxfrm. Note that it does
417 * NOT NULL terminate. That is left to the caller.
420 _collate_wxfrm(struct xlocale_collate *table, const wchar_t *src, wchar_t *xf,
429 const int32_t *state;
432 int ndir = table->info->directive_count;
436 for (pass = 0; pass <= ndir; pass++) {
441 /* insert level separator from the previous pass */
449 /* special pass for undefined */
451 direc = DIRECTIVE_FORWARD | DIRECTIVE_UNDEFINED;
453 direc = table->info->directive[pass];
458 if (direc & DIRECTIVE_BACKWARD) {
461 if ((tr = wcsdup(t)) == NULL) {
466 fp = tr + wcslen(tr) - 1;
472 t = (const wchar_t *)tr;
475 if (direc & DIRECTIVE_POSITION) {
476 while (*t || state) {
477 _collate_lookup(table, t, &len, &pri, pass, &state);
485 pri = COLLATE_MAX_PRIORITY;
495 while (*t || state) {
496 _collate_lookup(table, t, &len, &pri, pass, &state);
520 return ((size_t)(-1));
524 * In the non-POSIX case, we transform each character into a string of
525 * characters representing the character's priority. Since char is usually
526 * signed, we are limited by 7 bits per byte. To avoid zero, we need to add
527 * XFRM_OFFSET, so we can't use a full 7 bits. For simplicity, we choose 6
530 * It turns out that we sometimes have real priorities that are
531 * 31-bits wide. (But: be careful using priorities where the high
532 * order bit is set -- i.e. the priority is negative. The sort order
533 * may be surprising!)
535 * TODO: This would be a good area to optimize somewhat. It turns out
536 * that real prioririties *except for the last UNDEFINED pass* are generally
537 * very small. We need the localedef code to precalculate the max
538 * priority for us, and ideally also give us a mask, and then we could
539 * severely limit what we expand to.
542 #define XFRM_OFFSET ('0') /* make all printable characters */
544 #define XFRM_MASK ((1 << XFRM_SHIFT) - 1)
545 #define XFRM_SEP ('.') /* chosen to be less than XFRM_OFFSET */
548 xfrm(struct xlocale_collate *table, unsigned char *p, int pri, int pass)
550 /* we use unsigned to ensure zero fill on right shift */
551 uint32_t val = (uint32_t)table->info->pri_count[pass];
555 *p = (pri & XFRM_MASK) + XFRM_OFFSET;
565 _collate_sxfrm(struct xlocale_collate *table, const wchar_t *src, char *xf,
574 const int32_t *state;
578 uint8_t buf[XFRM_BYTES];
579 int ndir = table->info->directive_count;
583 for (pass = 0; pass <= ndir; pass++) {
588 /* insert level separator from the previous pass */
596 /* special pass for undefined */
598 direc = DIRECTIVE_FORWARD | DIRECTIVE_UNDEFINED;
600 direc = table->info->directive[pass];
605 if (direc & DIRECTIVE_BACKWARD) {
608 if ((tr = wcsdup(t)) == NULL) {
613 fp = tr + wcslen(tr) - 1;
619 t = (const wchar_t *)tr;
622 if (direc & DIRECTIVE_POSITION) {
623 while (*t || state) {
625 _collate_lookup(table, t, &len, &pri, pass, &state);
633 pri = COLLATE_MAX_PRIORITY;
636 b = xfrm(table, buf, pri, pass);
650 while (*t || state) {
651 _collate_lookup(table, t, &len, &pri, pass, &state);
662 b = xfrm(table, buf, pri, pass);
683 return ((size_t)(-1));
687 * __collate_equiv_value returns the primary collation value for the given
688 * collating symbol specified by str and len. Zero or negative is returned
689 * if the collating symbol was not found. This function is used by bracket
690 * code in the TRE regex library.
693 __collate_equiv_value(locale_t locale, const wchar_t *str, size_t len)
697 if (len < 1 || len >= COLLATE_STR_LEN)
701 struct xlocale_collate *table =
702 (struct xlocale_collate*)locale->components[XLC_COLLATE];
704 if (table->__collate_load_error)
705 return ((len == 1 && *str <= UCHAR_MAX) ? *str : -1);
709 if (*str <= UCHAR_MAX)
710 e = table->char_pri_table[*str].pri[0];
711 else if (table->info->large_count > 0) {
712 collate_large_t *match_large;
713 match_large = largesearch(table, *str);
715 e = match_large->pri.pri[0];
719 return (e > 0 ? e : 0);
721 if (table->info->chain_count > 0) {
722 wchar_t name[COLLATE_STR_LEN];
723 collate_chain_t *match_chain;
726 wcsncpy (name, str, len);
728 match_chain = chainsearch(table, name, &clen);
730 e = match_chain->pri[0];
733 return (e < 0 ? -e : e);