sys/libkern/iconv_ucs.c

   1 /*-
   2  * Copyright (c) 2003, 2005 Ryuichiro Imura
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/kernel.h>
  32 #include <sys/systm.h>
  33 #include <sys/malloc.h>
  34 #include <sys/iconv.h>
  35
  36 #include "iconv_converter_if.h"
  37
  38 /*
  39  * "UCS" converter
  40  */
  41
  42 #define KICONV_UCS_COMBINE      0x1
  43 #define KICONV_UCS_FROM_UTF8    0x2
  44 #define KICONV_UCS_TO_UTF8      0x4
  45 #define KICONV_UCS_FROM_LE      0x8
  46 #define KICONV_UCS_TO_LE        0x10
  47 #define KICONV_UCS_FROM_UTF16   0x20
  48 #define KICONV_UCS_TO_UTF16     0x40
  49 #define KICONV_UCS_UCS4         0x80
  50
  51 #define ENCODING_UTF16  "UTF-16BE"
  52 #define ENCODING_UTF8   "UTF-8"
  53
  54 static struct {
  55         const char *name;
  56         int from_flag, to_flag;
  57 } unicode_family[] = {
  58         { "UTF-8",      KICONV_UCS_FROM_UTF8,   KICONV_UCS_TO_UTF8 },
  59         { "UCS-2LE",    KICONV_UCS_FROM_LE,     KICONV_UCS_TO_LE },
  60         { "UTF-16BE",   KICONV_UCS_FROM_UTF16,  KICONV_UCS_TO_UTF16 },
  61         { "UTF-16LE",   KICONV_UCS_FROM_UTF16|KICONV_UCS_FROM_LE,
  62             KICONV_UCS_TO_UTF16|KICONV_UCS_TO_LE },
  63         { NULL,         0,      0 }
  64 };
  65
  66 static uint32_t utf8_to_ucs4(const char *src, size_t *utf8width, size_t srclen);
  67 static u_char *ucs4_to_utf8(uint32_t ucs4, char * dst, size_t *utf8width, size_t dstlen);
  68 static uint32_t encode_surrogate(uint32_t code);
  69 static uint32_t decode_surrogate(const u_char *ucs);
  70
  71 #ifdef MODULE_DEPEND
  72 MODULE_DEPEND(iconv_ucs, libiconv, 2, 2, 2);
  73 #endif
  74
  75 /*
  76  * UCS converter instance
  77  */
  78 struct iconv_ucs {
  79         KOBJ_FIELDS;
  80         int                     convtype;
  81         struct iconv_cspair *   d_csp;
  82         struct iconv_cspair *   d_cspf;
  83         void *                  f_ctp;
  84         void *                  t_ctp;
  85         void *                  ctype;
  86 };
  87
  88 static int
  89 iconv_ucs_open(struct iconv_converter_class *dcp,
  90         struct iconv_cspair *csp, struct iconv_cspair *cspf, void **dpp)
  91 {
  92         struct iconv_ucs *dp;
  93         int i;
  94         const char *from, *to;
  95
  96         dp = (struct iconv_ucs *)kobj_create((struct kobj_class*)dcp, M_ICONV, M_WAITOK);
  97         to = csp->cp_to;
  98         from = cspf ? cspf->cp_from : csp->cp_from;
  99
 100         dp->convtype = 0;
 101
 102         if (cspf)
 103                 dp->convtype |= KICONV_UCS_COMBINE;
 104         for (i = 0; unicode_family[i].name; i++) {
 105                 if (strcmp(from, unicode_family[i].name) == 0)
 106                         dp->convtype |= unicode_family[i].from_flag;
 107                 if (strcmp(to, unicode_family[i].name) == 0)
 108                         dp->convtype |= unicode_family[i].to_flag;
 109         }
 110         if (strcmp(ENCODING_UNICODE, ENCODING_UTF16) == 0)
 111                 dp->convtype |= KICONV_UCS_UCS4;
 112         else
 113                 dp->convtype &= ~KICONV_UCS_UCS4;
 114
 115         dp->f_ctp = dp->t_ctp = NULL;
 116         if (dp->convtype & KICONV_UCS_COMBINE) {
 117                 if ((dp->convtype & KICONV_UCS_FROM_UTF8) == 0 &&
 118                     (dp->convtype & KICONV_UCS_FROM_LE) == 0) {
 119                         iconv_open(ENCODING_UNICODE, from, &dp->f_ctp);
 120                 }
 121                 if ((dp->convtype & KICONV_UCS_TO_UTF8) == 0 &&
 122                     (dp->convtype & KICONV_UCS_TO_LE) == 0) {
 123                         iconv_open(to, ENCODING_UNICODE, &dp->t_ctp);
 124                 }
 125         }
 126
 127         dp->ctype = NULL;
 128         if (dp->convtype & (KICONV_UCS_FROM_UTF8 | KICONV_UCS_TO_UTF8))
 129                 iconv_open(KICONV_WCTYPE_NAME, ENCODING_UTF8, &dp->ctype);
 130
 131         dp->d_csp = csp;
 132         if (dp->convtype & (KICONV_UCS_FROM_UTF8 | KICONV_UCS_FROM_LE)) {
 133                 if (cspf) {
 134                         dp->d_cspf = cspf;
 135                         cspf->cp_refcount++;
 136                 } else
 137                         csp->cp_refcount++;
 138         }
 139         if (dp->convtype & (KICONV_UCS_TO_UTF8 | KICONV_UCS_TO_LE))
 140                 csp->cp_refcount++;
 141         *dpp = (void*)dp;
 142         return 0;
 143 }
 144
 145 static int
 146 iconv_ucs_close(void *data)
 147 {
 148         struct iconv_ucs *dp = data;
 149
 150         if (dp->f_ctp)
 151                 iconv_close(dp->f_ctp);
 152         if (dp->t_ctp)
 153                 iconv_close(dp->t_ctp);
 154         if (dp->ctype)
 155                 iconv_close(dp->ctype);
 156         if (dp->d_cspf)
 157                 dp->d_cspf->cp_refcount--;
 158         else if (dp->convtype & (KICONV_UCS_FROM_UTF8 | KICONV_UCS_FROM_LE))
 159                 dp->d_csp->cp_refcount--;
 160         if (dp->convtype & (KICONV_UCS_TO_UTF8 | KICONV_UCS_TO_LE))
 161                 dp->d_csp->cp_refcount--;
 162         kobj_delete((struct kobj*)data, M_ICONV);
 163         return 0;
 164 }
 165
 166 static int
 167 iconv_ucs_conv(void *d2p, const char **inbuf,
 168         size_t *inbytesleft, char **outbuf, size_t *outbytesleft,
 169         int convchar, int casetype)
 170 {
 171         struct iconv_ucs *dp = (struct iconv_ucs*)d2p;
 172         int ret = 0, i;
 173         size_t in, on, ir, or, inlen, outlen, ucslen;
 174         const char *src, *p;
 175         char *dst;
 176         u_char ucs[4], *q;
 177         uint32_t code;
 178
 179         if (inbuf == NULL || *inbuf == NULL || outbuf == NULL || *outbuf == NULL)
 180                 return 0;
 181         ir = in = *inbytesleft;
 182         or = on = *outbytesleft;
 183         src = *inbuf;
 184         dst = *outbuf;
 185
 186         while (ir > 0 && or > 0) {
 187
 188                 /*
 189                  * The first half of conversion.
 190                  * (convert any code into ENCODING_UNICODE)
 191                  */
 192                 code = 0;
 193                 p = src;
 194                 if (dp->convtype & KICONV_UCS_FROM_UTF8) {
 195                         /* convert UTF-8 to ENCODING_UNICODE */
 196                         inlen = 0;
 197                         code = utf8_to_ucs4(p, &inlen, ir);
 198                         if (code == 0) {
 199                                 ret = -1;
 200                                 break;
 201                         }
 202
 203                         if (casetype == KICONV_FROM_LOWER && dp->ctype) {
 204                                 code = towlower(code, dp->ctype);
 205                         } else if (casetype == KICONV_FROM_UPPER && dp->ctype) {
 206                                 code = towupper(code, dp->ctype);
 207                         }
 208
 209                         if ((code >= 0xd800 && code < 0xe000) || code >= 0x110000 ) {
 210                                 /* reserved for utf-16 surrogate pair */
 211                                 /* invalid unicode */
 212                                 ret = -1;
 213                                 break;
 214                         }
 215
 216                         if (inlen == 4) {
 217                                 if (dp->convtype & KICONV_UCS_UCS4) {
 218                                         ucslen = 4;
 219                                         code = encode_surrogate(code);
 220                                 } else {
 221                                         /* can't handle with ucs-2 */
 222                                         ret = -1;
 223                                         break;
 224                                 }
 225                         } else {
 226                                 ucslen = 2;
 227                         }
 228
 229                         /* save UCS-4 into ucs[] */
 230                         for (q = ucs, i = ucslen - 1 ; i >= 0 ; i--)
 231                                 *q++ = (code >> (i << 3)) & 0xff;
 232
 233                 } else if (dp->convtype & KICONV_UCS_COMBINE && dp->f_ctp) {
 234                         /* convert local code to ENCODING_UNICODE */
 235                         ucslen = 4;
 236                         inlen = ir;
 237                         q = ucs;
 238                         ret = iconv_convchr_case(dp->f_ctp, &p, &inlen, (char **)&q,
 239                             &ucslen, casetype & (KICONV_FROM_LOWER | KICONV_FROM_UPPER));
 240                         if (ret)
 241                                 break;
 242                         inlen = ir - inlen;
 243                         ucslen = 4 - ucslen;
 244
 245                 } else {
 246                         /* src code is a proper subset of ENCODING_UNICODE */
 247                         q = ucs;
 248                         if (dp->convtype & KICONV_UCS_FROM_LE) {
 249                                 *q = *(p + 1);
 250                                 *(q + 1) = *p;
 251                                 p += 2;
 252                         } else {
 253                                 *q = *p++;
 254                                 *(q + 1) = *p++;
 255                         }
 256                         if ((*q & 0xfc) == 0xd8) {
 257                                 if (dp->convtype & KICONV_UCS_UCS4 &&
 258                                     dp->convtype & KICONV_UCS_FROM_UTF16) {
 259                                         inlen = ucslen = 4;
 260                                 } else {
 261                                         /* invalid unicode */
 262                                         ret = -1;
 263                                         break;
 264                                 }
 265                         } else {
 266                                 inlen = ucslen = 2;
 267                         }
 268                         if (ir < inlen) {
 269                                 ret = -1;
 270                                 break;
 271                         }
 272                         if (ucslen == 4) {
 273                                 q += 2;
 274                                 if (dp->convtype & KICONV_UCS_FROM_LE) {
 275                                         *q = *(p + 1);
 276                                         *(q + 1) = *p;
 277                                 } else {
 278                                         *q = *p++;
 279                                         *(q + 1) = *p;
 280                                 }
 281                                 if ((*q & 0xfc) != 0xdc) {
 282                                         /* invalid unicode */
 283                                         ret = -1;
 284                                         break;
 285                                 }
 286                         }
 287                 }
 288
 289                 /*
 290                  * The second half of conversion.
 291                  * (convert ENCODING_UNICODE into any code)
 292                  */
 293                 p = ucs;
 294                 if (dp->convtype & KICONV_UCS_TO_UTF8) {
 295                         q = (u_char *)dst;
 296                         if (ucslen == 4 && dp->convtype & KICONV_UCS_UCS4) {
 297                                 /* decode surrogate pair */
 298                                 code = decode_surrogate(p);
 299                         } else {
 300                                 code = (ucs[0] << 8) | ucs[1];
 301                         }
 302
 303                         if (casetype == KICONV_LOWER && dp->ctype) {
 304                                 code = towlower(code, dp->ctype);
 305                         } else if (casetype == KICONV_UPPER && dp->ctype) {
 306                                 code = towupper(code, dp->ctype);
 307                         }
 308
 309                         outlen = 0;
 310                         if (ucs4_to_utf8(code, q, &outlen, or) == NULL) {
 311                                 ret = -1;
 312                                 break;
 313                         }
 314
 315                         src += inlen;
 316                         ir -= inlen;
 317                         dst += outlen;
 318                         or -= outlen;
 319
 320                 } else if (dp->convtype & KICONV_UCS_COMBINE && dp->t_ctp) {
 321                         ret = iconv_convchr_case(dp->t_ctp, &p, &ucslen, &dst,
 322                             &or, casetype & (KICONV_LOWER | KICONV_UPPER));
 323                         if (ret)
 324                                 break;
 325
 326                         src += inlen;
 327                         ir -= inlen;
 328
 329                 } else {
 330                         /* dst code is a proper subset of ENCODING_UNICODE */
 331                         if (or < ucslen) {
 332                                 ret = -1;
 333                                 break;
 334                         }
 335                         src += inlen;
 336                         ir -= inlen;
 337                         or -= ucslen;
 338                         if (dp->convtype & KICONV_UCS_TO_LE) {
 339                                 *dst++ = *(p + 1);
 340                                 *dst++ = *p;
 341                                 p += 2;
 342                         } else {
 343                                 *dst++ = *p++;
 344                                 *dst++ = *p++;
 345                         }
 346                         if (ucslen == 4) {
 347                                 if ((dp->convtype & KICONV_UCS_UCS4) == 0 ||
 348                                     (dp->convtype & KICONV_UCS_TO_UTF16) == 0) {
 349                                         ret = -1;
 350                                         break;
 351                                 }
 352                                 if (dp->convtype & KICONV_UCS_TO_LE) {
 353                                         *dst++ = *(p + 1);
 354                                         *dst++ = *p;
 355                                 } else {
 356                                         *dst++ = *p++;
 357                                         *dst++ = *p;
 358                                 }
 359                         }
 360                 }
 361
 362                 if (convchar == 1)
 363                         break;
 364         }
 365
 366         *inbuf += in - ir;
 367         *outbuf += on - or;
 368         *inbytesleft -= in - ir;
 369         *outbytesleft -= on - or;
 370         return (ret);
 371 }
 372
 373 static int
 374 iconv_ucs_init(struct iconv_converter_class *dcp)
 375 {
 376         int error;
 377
 378         error = iconv_add(ENCODING_UNICODE, ENCODING_UNICODE, ENCODING_UTF8);
 379         if (error)
 380                 return (error);
 381         error = iconv_add(ENCODING_UNICODE, ENCODING_UTF8, ENCODING_UNICODE);
 382         if (error)
 383                 return (error);
 384         return (0);
 385 }
 386
 387 static int
 388 iconv_ucs_done(struct iconv_converter_class *dcp)
 389 {
 390         return (0);
 391 }
 392
 393 static const char *
 394 iconv_ucs_name(struct iconv_converter_class *dcp)
 395 {
 396         return (ENCODING_UNICODE);
 397 }
 398
 399 static kobj_method_t iconv_ucs_methods[] = {
 400         KOBJMETHOD(iconv_converter_open,        iconv_ucs_open),
 401         KOBJMETHOD(iconv_converter_close,       iconv_ucs_close),
 402         KOBJMETHOD(iconv_converter_conv,        iconv_ucs_conv),
 403         KOBJMETHOD(iconv_converter_init,        iconv_ucs_init),
 404         KOBJMETHOD(iconv_converter_done,        iconv_ucs_done),
 405         KOBJMETHOD(iconv_converter_name,        iconv_ucs_name),
 406         {0, 0}
 407 };
 408
 409 KICONV_CONVERTER(ucs, sizeof(struct iconv_ucs));
 410
 411 static uint32_t
 412 utf8_to_ucs4(const char *src, size_t *utf8width, size_t srclen)
 413 {
 414         size_t i, w = 0;
 415         uint32_t ucs4 = 0;
 416
 417         /*
 418          * get leading 1 byte from utf-8
 419          */
 420         if ((*src & 0x80) == 0) {
 421                 /*
 422                  * leading 1 bit is "0"
 423                  *  utf-8: 0xxxxxxx
 424                  *  ucs-4: 00000000 00000000 00000000 0xxxxxxx
 425                  */
 426                 w = 1;
 427                 /* get trailing 7 bits */
 428                 ucs4 = *src & 0x7f;
 429         } else if ((*src & 0xe0) == 0xc0) {
 430                 /*
 431                  * leading 3 bits are "110"
 432                  *  utf-8: 110xxxxx 10yyyyyy
 433                  *  ucs-4: 00000000 00000000 00000xxx xxyyyyyy
 434                  */
 435                 w = 2;
 436                 /* get trailing 5 bits */
 437                 ucs4 = *src & 0x1f;
 438         } else if ((*src & 0xf0) == 0xe0) {
 439                 /*
 440                  * leading 4 bits are "1110"
 441                  *  utf-8: 1110xxxx 10yyyyyy 10zzzzzz
 442                  *  ucs-4: 00000000 00000000 xxxxyyyy yyzzzzzz
 443                  */
 444                 w = 3;
 445                 /* get trailing 4 bits */
 446                 ucs4 = *src & 0x0f;
 447         } else if ((*src & 0xf8) == 0xf0) {
 448                 /*
 449                  * leading 5 bits are "11110"
 450                  *  utf-8: 11110www 10xxxxxx 10yyyyyy 10zzzzzz
 451                  *  ucs-4: 00000000 000wwwxx xxxxyyyy yyzzzzzz
 452                  */
 453                 w = 4;
 454                 /* get trailing 3 bits */
 455                 ucs4 = *src & 0x07;
 456         } else {
 457                 /* out of utf-16 range or having illegal bits */
 458                 return (0);
 459         }
 460
 461         if (srclen < w)
 462                 return (0);
 463
 464         /*
 465          * get left parts from utf-8
 466          */
 467         for (i = 1 ; i < w ; i++) {
 468                 if ((*(src + i) & 0xc0) != 0x80) {
 469                         /* invalid: leading 2 bits are not "10" */
 470                         return (0);
 471                 }
 472                 /* concatenate trailing 6 bits into ucs4 */
 473                 ucs4 <<= 6;
 474                 ucs4 |= *(src + i) & 0x3f;
 475         }
 476
 477         *utf8width = w;
 478         return (ucs4);
 479 }
 480
 481 static u_char *
 482 ucs4_to_utf8(uint32_t ucs4, char *dst, size_t *utf8width, size_t dstlen)
 483 {
 484         u_char lead, *p;
 485         size_t i, w;
 486
 487         /*
 488          * determine utf-8 width and leading bits
 489          */
 490         if (ucs4 < 0x80) {
 491                 w = 1;
 492                 lead = 0;       /* "0" */
 493         } else if (ucs4 < 0x800) {
 494                 w = 2;
 495                 lead = 0xc0;    /* "11" */
 496         } else if (ucs4 < 0x10000) {
 497                 w = 3;
 498                 lead = 0xe0;    /* "111" */
 499         } else if (ucs4 < 0x200000) {
 500                 w = 4;
 501                 lead = 0xf0;    /* "1111" */
 502         } else {
 503                 return (NULL);
 504         }
 505
 506         if (dstlen < w)
 507                 return (NULL);
 508
 509         /*
 510          * construct utf-8
 511          */
 512         p = dst;
 513         for (i = w - 1 ; i >= 1 ; i--) {
 514                 /* get trailing 6 bits and put it with leading bit as "1" */
 515                 *(p + i) = (ucs4 & 0x3f) | 0x80;
 516                 ucs4 >>= 6;
 517         }
 518         *p = ucs4 | lead;
 519
 520         *utf8width = w;
 521
 522         return (p);
 523 }
 524
 525 static uint32_t
 526 encode_surrogate(register uint32_t code)
 527 {
 528         return ((((code - 0x10000) << 6) & 0x3ff0000) |
 529             ((code - 0x10000) & 0x3ff) | 0xd800dc00);
 530 }
 531
 532 static uint32_t
 533 decode_surrogate(register const u_char *ucs)
 534 {
 535         return ((((ucs[0] & 0x3) << 18) | (ucs[1] << 10) |
 536             ((ucs[2] & 0x3) << 8) | ucs[3]) + 0x10000);
 537 }
 538