usr.bin/sort/bwstring.c

   1 /*-
   2  * Copyright (C) 2009 Gabor Kovesdan <gabor@FreeBSD.org>
   3  * Copyright (C) 2012 Oleg Moskalenko <mom040267@gmail.com>
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include <ctype.h>
  32 #include <errno.h>
  33 #include <err.h>
  34 #include <langinfo.h>
  35 #include <math.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <wchar.h>
  39 #include <wctype.h>
  40
  41 #include "bwstring.h"
  42 #include "sort.h"
  43
  44 bool byte_sort;
  45
  46 static wchar_t **wmonths;
  47 static unsigned char **cmonths;
  48
  49 /* initialise months */
  50
  51 void
  52 initialise_months(void)
  53 {
  54         const nl_item item[12] = { ABMON_1, ABMON_2, ABMON_3, ABMON_4,
  55             ABMON_5, ABMON_6, ABMON_7, ABMON_8, ABMON_9, ABMON_10,
  56             ABMON_11, ABMON_12 };
  57         unsigned char *tmp;
  58         size_t len;
  59
  60         if (MB_CUR_MAX == 1) {
  61                 if (cmonths == NULL) {
  62                         unsigned char *m;
  63
  64                         cmonths = sort_malloc(sizeof(unsigned char*) * 12);
  65                         for (int i = 0; i < 12; i++) {
  66                                 cmonths[i] = NULL;
  67                                 tmp = (unsigned char *) nl_langinfo(item[i]);
  68                                 if (debug_sort)
  69                                         printf("month[%d]=%s\n", i, tmp);
  70                                 if (*tmp == '\0')
  71                                         continue;
  72                                 m = sort_strdup(tmp);
  73                                 len = strlen(tmp);
  74                                 for (unsigned int j = 0; j < len; j++)
  75                                         m[j] = toupper(m[j]);
  76                                 cmonths[i] = m;
  77                         }
  78                 }
  79
  80         } else {
  81                 if (wmonths == NULL) {
  82                         wchar_t *m;
  83
  84                         wmonths = sort_malloc(sizeof(wchar_t *) * 12);
  85                         for (int i = 0; i < 12; i++) {
  86                                 wmonths[i] = NULL;
  87                                 tmp = (unsigned char *) nl_langinfo(item[i]);
  88                                 if (debug_sort)
  89                                         printf("month[%d]=%s\n", i, tmp);
  90                                 if (*tmp == '\0')
  91                                         continue;
  92                                 len = strlen(tmp);
  93                                 m = sort_malloc(SIZEOF_WCHAR_STRING(len + 1));
  94                                 if (mbstowcs(m, (char*)tmp, len) ==
  95                                     ((size_t) - 1)) {
  96                                         sort_free(m);
  97                                         continue;
  98                                 }
  99                                 m[len] = L'\0';
 100                                 for (unsigned int j = 0; j < len; j++)
 101                                         m[j] = towupper(m[j]);
 102                                 wmonths[i] = m;
 103                         }
 104                 }
 105         }
 106 }
 107
 108 /*
 109  * Compare two wide-character strings
 110  */
 111 static int
 112 wide_str_coll(const wchar_t *s1, const wchar_t *s2)
 113 {
 114         int ret = 0;
 115
 116         errno = 0;
 117         ret = wcscoll(s1, s2);
 118         if (errno == EILSEQ) {
 119                 errno = 0;
 120                 ret = wcscmp(s1, s2);
 121                 if (errno != 0) {
 122                         for (size_t i = 0; ; ++i) {
 123                                 wchar_t c1 = s1[i];
 124                                 wchar_t c2 = s2[i];
 125                                 if (c1 == L'\0')
 126                                         return ((c2 == L'\0') ? 0 : -1);
 127                                 if (c2 == L'\0')
 128                                         return (+1);
 129                                 if (c1 == c2)
 130                                         continue;
 131                                 return ((int)(c1 - c2));
 132                         }
 133                 }
 134         }
 135         return (ret);
 136 }
 137
 138 /* counterparts of wcs functions */
 139
 140 void
 141 bwsprintf(FILE *f, struct bwstring *bws, const char *prefix, const char *suffix)
 142 {
 143
 144         if (MB_CUR_MAX == 1)
 145                 fprintf(f, "%s%s%s", prefix, bws->data.cstr, suffix);
 146         else
 147                 fprintf(f, "%s%S%s", prefix, bws->data.wstr, suffix);
 148 }
 149
 150 const void* bwsrawdata(const struct bwstring *bws)
 151 {
 152
 153         return (&(bws->data));
 154 }
 155
 156 size_t bwsrawlen(const struct bwstring *bws)
 157 {
 158
 159         return ((MB_CUR_MAX == 1) ? bws->len : SIZEOF_WCHAR_STRING(bws->len));
 160 }
 161
 162 size_t
 163 bws_memsize(const struct bwstring *bws)
 164 {
 165
 166         return ((MB_CUR_MAX == 1) ? (bws->len + 2 + sizeof(struct bwstring)) :
 167             (SIZEOF_WCHAR_STRING(bws->len + 1) + sizeof(struct bwstring)));
 168 }
 169
 170 void
 171 bws_setlen(struct bwstring *bws, size_t newlen)
 172 {
 173
 174         if (bws && newlen != bws->len && newlen <= bws->len) {
 175                 bws->len = newlen;
 176                 if (MB_CUR_MAX == 1)
 177                         bws->data.cstr[newlen] = '\0';
 178                 else
 179                         bws->data.wstr[newlen] = L'\0';
 180         }
 181 }
 182
 183 /*
 184  * Allocate a new binary string of specified size
 185  */
 186 struct bwstring *
 187 bwsalloc(size_t sz)
 188 {
 189         struct bwstring *ret;
 190
 191         if (MB_CUR_MAX == 1)
 192                 ret = sort_malloc(sizeof(struct bwstring) + 1 + sz);
 193         else
 194                 ret = sort_malloc(sizeof(struct bwstring) +
 195                     SIZEOF_WCHAR_STRING(sz + 1));
 196         ret->len = sz;
 197
 198         if (MB_CUR_MAX == 1)
 199                 ret->data.cstr[ret->len] = '\0';
 200         else
 201                 ret->data.wstr[ret->len] = L'\0';
 202
 203         return (ret);
 204 }
 205
 206 /*
 207  * Create a copy of binary string.
 208  * New string size equals the length of the old string.
 209  */
 210 struct bwstring *
 211 bwsdup(const struct bwstring *s)
 212 {
 213
 214         if (s == NULL)
 215                 return (NULL);
 216         else {
 217                 struct bwstring *ret = bwsalloc(s->len);
 218
 219                 if (MB_CUR_MAX == 1)
 220                         memcpy(ret->data.cstr, s->data.cstr, (s->len));
 221                 else
 222                         memcpy(ret->data.wstr, s->data.wstr,
 223                             SIZEOF_WCHAR_STRING(s->len));
 224
 225                 return (ret);
 226         }
 227 }
 228
 229 /*
 230  * Create a new binary string from a raw binary buffer.
 231  */
 232 struct bwstring *
 233 bwssbdup(const wchar_t *str, size_t len)
 234 {
 235
 236         if (str == NULL)
 237                 return ((len == 0) ? bwsalloc(0) : NULL);
 238         else {
 239                 struct bwstring *ret;
 240
 241                 ret = bwsalloc(len);
 242
 243                 if (MB_CUR_MAX == 1)
 244                         for (size_t i = 0; i < len; ++i)
 245                                 ret->data.cstr[i] = (unsigned char) str[i];
 246                 else
 247                         memcpy(ret->data.wstr, str, SIZEOF_WCHAR_STRING(len));
 248
 249                 return (ret);
 250         }
 251 }
 252
 253 /*
 254  * Create a new binary string from a raw binary buffer.
 255  */
 256 struct bwstring *
 257 bwscsbdup(const unsigned char *str, size_t len)
 258 {
 259         struct bwstring *ret;
 260
 261         ret = bwsalloc(len);
 262
 263         if (str) {
 264                 if (MB_CUR_MAX == 1)
 265                         memcpy(ret->data.cstr, str, len);
 266                 else {
 267                         mbstate_t mbs;
 268                         const char *s;
 269                         size_t charlen, chars, cptr;
 270
 271                         charlen = chars = 0;
 272                         cptr = 0;
 273                         s = (const char *) str;
 274
 275                         memset(&mbs, 0, sizeof(mbs));
 276
 277                         while (cptr < len) {
 278                                 size_t n = MB_CUR_MAX;
 279
 280                                 if (n > len - cptr)
 281                                         n = len - cptr;
 282                                 charlen = mbrlen(s + cptr, n, &mbs);
 283                                 switch (charlen) {
 284                                 case 0:
 285                                         /* FALLTHROUGH */
 286                                 case (size_t) -1:
 287                                         /* FALLTHROUGH */
 288                                 case (size_t) -2:
 289                                         ret->data.wstr[chars++] =
 290                                             (unsigned char) s[cptr];
 291                                         ++cptr;
 292                                         break;
 293                                 default:
 294                                         n = mbrtowc(ret->data.wstr + (chars++),
 295                                             s + cptr, charlen, &mbs);
 296                                         if ((n == (size_t)-1) || (n == (size_t)-2))
 297                                                 /* NOTREACHED */
 298                                                 err(2, "mbrtowc error");
 299                                         cptr += charlen;
 300                                 };
 301                         }
 302
 303                         ret->len = chars;
 304                         ret->data.wstr[ret->len] = L'\0';
 305                 }
 306         }
 307         return (ret);
 308 }
 309
 310 /*
 311  * De-allocate object memory
 312  */
 313 void
 314 bwsfree(const struct bwstring *s)
 315 {
 316
 317         if (s)
 318                 sort_free(s);
 319 }
 320
 321 /*
 322  * Copy content of src binary string to dst.
 323  * If the capacity of the dst string is not sufficient,
 324  * then the data is truncated.
 325  */
 326 size_t
 327 bwscpy(struct bwstring *dst, const struct bwstring *src)
 328 {
 329         size_t nums = src->len;
 330
 331         if (nums > dst->len)
 332                 nums = dst->len;
 333         dst->len = nums;
 334
 335         if (MB_CUR_MAX == 1) {
 336                 memcpy(dst->data.cstr, src->data.cstr, nums);
 337                 dst->data.cstr[dst->len] = '\0';
 338         } else {
 339                 memcpy(dst->data.wstr, src->data.wstr,
 340                     SIZEOF_WCHAR_STRING(nums + 1));
 341                 dst->data.wstr[dst->len] = L'\0';
 342         }
 343
 344         return (nums);
 345 }
 346
 347 /*
 348  * Copy content of src binary string to dst,
 349  * with specified number of symbols to be copied.
 350  * If the capacity of the dst string is not sufficient,
 351  * then the data is truncated.
 352  */
 353 struct bwstring *
 354 bwsncpy(struct bwstring *dst, const struct bwstring *src, size_t size)
 355 {
 356         size_t nums = src->len;
 357
 358         if (nums > dst->len)
 359                 nums = dst->len;
 360         if (nums > size)
 361                 nums = size;
 362         dst->len = nums;
 363
 364         if (MB_CUR_MAX == 1) {
 365                 memcpy(dst->data.cstr, src->data.cstr, nums);
 366                 dst->data.cstr[dst->len] = '\0';
 367         } else {
 368                 memcpy(dst->data.wstr, src->data.wstr,
 369                     SIZEOF_WCHAR_STRING(nums + 1));
 370                 dst->data.wstr[dst->len] = L'\0';
 371         }
 372
 373         return (dst);
 374 }
 375
 376 /*
 377  * Copy content of src binary string to dst,
 378  * with specified number of symbols to be copied.
 379  * An offset value can be specified, from the start of src string.
 380  * If the capacity of the dst string is not sufficient,
 381  * then the data is truncated.
 382  */
 383 struct bwstring *
 384 bwsnocpy(struct bwstring *dst, const struct bwstring *src, size_t offset,
 385     size_t size)
 386 {
 387
 388         if (offset >= src->len) {
 389                 dst->data.wstr[0] = 0;
 390                 dst->len = 0;
 391         } else {
 392                 size_t nums = src->len - offset;
 393
 394                 if (nums > dst->len)
 395                         nums = dst->len;
 396                 if (nums > size)
 397                         nums = size;
 398                 dst->len = nums;
 399                 if (MB_CUR_MAX == 1) {
 400                         memcpy(dst->data.cstr, src->data.cstr + offset,
 401                             (nums));
 402                         dst->data.cstr[dst->len] = '\0';
 403                 } else {
 404                         memcpy(dst->data.wstr, src->data.wstr + offset,
 405                             SIZEOF_WCHAR_STRING(nums));
 406                         dst->data.wstr[dst->len] = L'\0';
 407                 }
 408         }
 409         return (dst);
 410 }
 411
 412 /*
 413  * Write binary string to the file.
 414  * The output is ended either with '\n' (nl == true)
 415  * or '\0' (nl == false).
 416  */
 417 size_t
 418 bwsfwrite(struct bwstring *bws, FILE *f, bool zero_ended)
 419 {
 420
 421         if (MB_CUR_MAX == 1) {
 422                 size_t len = bws->len;
 423
 424                 if (!zero_ended) {
 425                         bws->data.cstr[len] = '\n';
 426
 427                         if (fwrite(bws->data.cstr, len + 1, 1, f) < 1)
 428                                 err(2, NULL);
 429
 430                         bws->data.cstr[len] = '\0';
 431                 } else if (fwrite(bws->data.cstr, len + 1, 1, f) < 1)
 432                         err(2, NULL);
 433
 434                 return (len + 1);
 435
 436         } else {
 437                 wchar_t eols;
 438                 size_t printed = 0;
 439
 440                 eols = zero_ended ? btowc('\0') : btowc('\n');
 441
 442                 while (printed < BWSLEN(bws)) {
 443                         const wchar_t *s = bws->data.wstr + printed;
 444
 445                         if (*s == L'\0') {
 446                                 int nums;
 447
 448                                 nums = fwprintf(f, L"%lc", *s);
 449
 450                                 if (nums != 1)
 451                                         err(2, NULL);
 452                                 ++printed;
 453                         } else {
 454                                 int nums;
 455
 456                                 nums = fwprintf(f, L"%ls", s);
 457
 458                                 if (nums < 1)
 459                                         err(2, NULL);
 460                                 printed += nums;
 461                         }
 462                 }
 463                 fwprintf(f, L"%lc", eols);
 464                 return (printed + 1);
 465         }
 466 }
 467
 468 /*
 469  * Allocate and read a binary string from file.
 470  * The strings are nl-ended or zero-ended, depending on the sort setting.
 471  */
 472 struct bwstring *
 473 bwsfgetln(FILE *f, size_t *len, bool zero_ended, struct reader_buffer *rb)
 474 {
 475         wint_t eols;
 476
 477         eols = zero_ended ? btowc('\0') : btowc('\n');
 478
 479         if (!zero_ended && (MB_CUR_MAX > 1)) {
 480                 wchar_t *ret;
 481
 482                 ret = fgetwln(f, len);
 483
 484                 if (ret == NULL) {
 485                         if (!feof(f))
 486                                 err(2, NULL);
 487                         return (NULL);
 488                 }
 489                 if (*len > 0) {
 490                         if (ret[*len - 1] == (wchar_t)eols)
 491                                 --(*len);
 492                 }
 493                 return (bwssbdup(ret, *len));
 494
 495         } else if (!zero_ended && (MB_CUR_MAX == 1)) {
 496                 char *ret;
 497
 498                 ret = fgetln(f, len);
 499
 500                 if (ret == NULL) {
 501                         if (!feof(f))
 502                                 err(2, NULL);
 503                         return (NULL);
 504                 }
 505                 if (*len > 0) {
 506                         if (ret[*len - 1] == '\n')
 507                                 --(*len);
 508                 }
 509                 return (bwscsbdup((unsigned char*)ret, *len));
 510
 511         } else {
 512                 *len = 0;
 513
 514                 if (feof(f))
 515                         return (NULL);
 516
 517                 if (2 >= rb->fgetwln_z_buffer_size) {
 518                         rb->fgetwln_z_buffer_size += 256;
 519                         rb->fgetwln_z_buffer = sort_realloc(rb->fgetwln_z_buffer,
 520                             sizeof(wchar_t) * rb->fgetwln_z_buffer_size);
 521                 }
 522                 rb->fgetwln_z_buffer[*len] = 0;
 523
 524                 if (MB_CUR_MAX == 1)
 525                         while (!feof(f)) {
 526                                 int c;
 527
 528                                 c = fgetc(f);
 529
 530                                 if (c == EOF) {
 531                                         if (*len == 0)
 532                                                 return (NULL);
 533                                         goto line_read_done;
 534                                 }
 535                                 if (c == eols)
 536                                         goto line_read_done;
 537
 538                                 if (*len + 1 >= rb->fgetwln_z_buffer_size) {
 539                                         rb->fgetwln_z_buffer_size += 256;
 540                                         rb->fgetwln_z_buffer = sort_realloc(rb->fgetwln_z_buffer,
 541                                             SIZEOF_WCHAR_STRING(rb->fgetwln_z_buffer_size));
 542                                 }
 543
 544                                 rb->fgetwln_z_buffer[*len] = c;
 545                                 rb->fgetwln_z_buffer[++(*len)] = 0;
 546                         }
 547                 else
 548                         while (!feof(f)) {
 549                                 wint_t c = 0;
 550
 551                                 c = fgetwc(f);
 552
 553                                 if (c == WEOF) {
 554                                         if (*len == 0)
 555                                                 return (NULL);
 556                                         goto line_read_done;
 557                                 }
 558                                 if (c == eols)
 559                                         goto line_read_done;
 560
 561                                 if (*len + 1 >= rb->fgetwln_z_buffer_size) {
 562                                         rb->fgetwln_z_buffer_size += 256;
 563                                         rb->fgetwln_z_buffer = sort_realloc(rb->fgetwln_z_buffer,
 564                                             SIZEOF_WCHAR_STRING(rb->fgetwln_z_buffer_size));
 565                                 }
 566
 567                                 rb->fgetwln_z_buffer[*len] = c;
 568                                 rb->fgetwln_z_buffer[++(*len)] = 0;
 569                         }
 570
 571 line_read_done:
 572                 /* we do not count the last 0 */
 573                 return (bwssbdup(rb->fgetwln_z_buffer, *len));
 574         }
 575 }
 576
 577 int
 578 bwsncmp(const struct bwstring *bws1, const struct bwstring *bws2,
 579     size_t offset, size_t len)
 580 {
 581         size_t cmp_len, len1, len2;
 582         int res = 0;
 583
 584         cmp_len = 0;
 585         len1 = bws1->len;
 586         len2 = bws2->len;
 587
 588         if (len1 <= offset) {
 589                 return ((len2 <= offset) ? 0 : -1);
 590         } else {
 591                 if (len2 <= offset)
 592                         return (+1);
 593                 else {
 594                         len1 -= offset;
 595                         len2 -= offset;
 596
 597                         cmp_len = len1;
 598
 599                         if (len2 < cmp_len)
 600                                 cmp_len = len2;
 601
 602                         if (len < cmp_len)
 603                                 cmp_len = len;
 604
 605                         if (MB_CUR_MAX == 1) {
 606                                 const unsigned char *s1, *s2;
 607
 608                                 s1 = bws1->data.cstr + offset;
 609                                 s2 = bws2->data.cstr + offset;
 610
 611                                 res = memcmp(s1, s2, cmp_len);
 612
 613                         } else {
 614                                 const wchar_t *s1, *s2;
 615
 616                                 s1 = bws1->data.wstr + offset;
 617                                 s2 = bws2->data.wstr + offset;
 618
 619                                 res = memcmp(s1, s2, SIZEOF_WCHAR_STRING(cmp_len));
 620                         }
 621                 }
 622         }
 623
 624         if (res == 0) {
 625                 if (len1 < cmp_len && len1 < len2)
 626                         res = -1;
 627                 else if (len2 < cmp_len && len2 < len1)
 628                         res = +1;
 629         }
 630
 631         return (res);
 632 }
 633
 634 int
 635 bwscmp(const struct bwstring *bws1, const struct bwstring *bws2, size_t offset)
 636 {
 637         size_t len1, len2, cmp_len;
 638         int res;
 639
 640         len1 = bws1->len;
 641         len2 = bws2->len;
 642
 643         len1 -= offset;
 644         len2 -= offset;
 645
 646         cmp_len = len1;
 647
 648         if (len2 < cmp_len)
 649                 cmp_len = len2;
 650
 651         res = bwsncmp(bws1, bws2, offset, cmp_len);
 652
 653         if (res == 0) {
 654                 if( len1 < len2)
 655                         res = -1;
 656                 else if (len2 < len1)
 657                         res = +1;
 658         }
 659
 660         return (res);
 661 }
 662
 663 int
 664 bws_iterator_cmp(bwstring_iterator iter1, bwstring_iterator iter2, size_t len)
 665 {
 666         wchar_t c1, c2;
 667         size_t i = 0;
 668
 669         for (i = 0; i < len; ++i) {
 670                 c1 = bws_get_iter_value(iter1);
 671                 c2 = bws_get_iter_value(iter2);
 672                 if (c1 != c2)
 673                         return (c1 - c2);
 674                 iter1 = bws_iterator_inc(iter1, 1);
 675                 iter2 = bws_iterator_inc(iter2, 1);
 676         }
 677
 678         return (0);
 679 }
 680
 681 int
 682 bwscoll(const struct bwstring *bws1, const struct bwstring *bws2, size_t offset)
 683 {
 684         size_t len1, len2;
 685
 686         len1 = bws1->len;
 687         len2 = bws2->len;
 688
 689         if (len1 <= offset)
 690                 return ((len2 <= offset) ? 0 : -1);
 691         else {
 692                 if (len2 <= offset)
 693                         return (+1);
 694                 else {
 695                         len1 -= offset;
 696                         len2 -= offset;
 697
 698                         if (MB_CUR_MAX == 1) {
 699                                 const unsigned char *s1, *s2;
 700
 701                                 s1 = bws1->data.cstr + offset;
 702                                 s2 = bws2->data.cstr + offset;
 703
 704                                 if (byte_sort) {
 705                                         int res = 0;
 706
 707                                         if (len1 > len2) {
 708                                                 res = memcmp(s1, s2, len2);
 709                                                 if (!res)
 710                                                         res = +1;
 711                                         } else if (len1 < len2) {
 712                                                 res = memcmp(s1, s2, len1);
 713                                                 if (!res)
 714                                                         res = -1;
 715                                         } else
 716                                                 res = memcmp(s1, s2, len1);
 717
 718                                         return (res);
 719
 720                                 } else {
 721                                         int res = 0;
 722                                         size_t i, maxlen;
 723
 724                                         i = 0;
 725                                         maxlen = len1;
 726
 727                                         if (maxlen > len2)
 728                                                 maxlen = len2;
 729
 730                                         while (i < maxlen) {
 731                                                 /* goto next non-zero part: */
 732                                                 while ((i < maxlen) &&
 733                                                     !s1[i] && !s2[i])
 734                                                         ++i;
 735
 736                                                 if (i >= maxlen)
 737                                                         break;
 738
 739                                                 if (s1[i] == 0) {
 740                                                         if (s2[i] == 0)
 741                                                                 /* NOTREACHED */
 742                                                                 err(2, "bwscoll error 01");
 743                                                         else
 744                                                                 return (-1);
 745                                                 } else if (s2[i] == 0)
 746                                                         return (+1);
 747
 748                                                 res = strcoll((const char*)(s1 + i), (const char*)(s2 + i));
 749                                                 if (res)
 750                                                         return (res);
 751
 752                                                 while ((i < maxlen) &&
 753                                                     s1[i] && s2[i])
 754                                                         ++i;
 755
 756                                                 if (i >= maxlen)
 757                                                         break;
 758
 759                                                 if (s1[i] == 0) {
 760                                                         if (s2[i] == 0) {
 761                                                                 ++i;
 762                                                                 continue;
 763                                                         } else
 764                                                                 return (-1);
 765                                                 } else if (s2[i] == 0)
 766                                                         return (+1);
 767                                                 else
 768                                                         /* NOTREACHED */
 769                                                         err(2, "bwscoll error 02");
 770                                         }
 771
 772                                         if (len1 < len2)
 773                                                 return (-1);
 774                                         else if (len1 > len2)
 775                                                 return (+1);
 776
 777                                         return (0);
 778                                 }
 779                         } else {
 780                                 const wchar_t *s1, *s2;
 781                                 size_t i, maxlen;
 782                                 int res = 0;
 783
 784                                 s1 = bws1->data.wstr + offset;
 785                                 s2 = bws2->data.wstr + offset;
 786
 787                                 i = 0;
 788                                 maxlen = len1;
 789
 790                                 if (maxlen > len2)
 791                                         maxlen = len2;
 792
 793                                 while (i < maxlen) {
 794
 795                                         /* goto next non-zero part: */
 796                                         while ((i < maxlen) &&
 797                                             !s1[i] && !s2[i])
 798                                                 ++i;
 799
 800                                         if (i >= maxlen)
 801                                                 break;
 802
 803                                         if (s1[i] == 0) {
 804                                                 if (s2[i] == 0)
 805                                                         /* NOTREACHED */
 806                                                         err(2, "bwscoll error 1");
 807                                                 else
 808                                                         return (-1);
 809                                         } else if (s2[i] == 0)
 810                                                 return (+1);
 811
 812                                         res = wide_str_coll(s1 + i, s2 + i);
 813                                         if (res)
 814                                                 return (res);
 815
 816                                         while ((i < maxlen) && s1[i] && s2[i])
 817                                                 ++i;
 818
 819                                         if (i >= maxlen)
 820                                                 break;
 821
 822                                         if (s1[i] == 0) {
 823                                                 if (s2[i] == 0) {
 824                                                         ++i;
 825                                                         continue;
 826                                                 } else
 827                                                         return (-1);
 828                                         } else if (s2[i] == 0)
 829                                                 return (+1);
 830                                         else
 831                                                 /* NOTREACHED */
 832                                                 err(2, "bwscoll error 2");
 833                                 }
 834
 835                                 if (len1 < len2)
 836                                         return (-1);
 837                                 else if (len1 > len2)
 838                                         return (+1);
 839
 840                                 return (0);
 841                         }
 842                 }
 843         }
 844 }
 845
 846 /*
 847  * Correction of the system API
 848  */
 849 double
 850 bwstod(struct bwstring *s0, bool *empty)
 851 {
 852         double ret = 0;
 853
 854         if (MB_CUR_MAX == 1) {
 855                 unsigned char *end, *s;
 856                 char *ep;
 857
 858                 s = s0->data.cstr;
 859                 end = s + s0->len;
 860                 ep = NULL;
 861
 862                 while (isblank(*s) && s < end)
 863                         ++s;
 864
 865                 if (!isprint(*s)) {
 866                         *empty = true;
 867                         return (0);
 868                 }
 869
 870                 ret = strtod((char*)s, &ep);
 871                 if ((unsigned char*) ep == s) {
 872                         *empty = true;
 873                         return (0);
 874                 }
 875         } else {
 876                 wchar_t *end, *ep, *s;
 877
 878                 s = s0->data.wstr;
 879                 end = s + s0->len;
 880                 ep = NULL;
 881
 882                 while (iswblank(*s) && s < end)
 883                         ++s;
 884
 885                 if (!iswprint(*s)) {
 886                         *empty = true;
 887                         return (0);
 888                 }
 889
 890                 ret = wcstod(s, &ep);
 891                 if (ep == s) {
 892                         *empty = true;
 893                         return (0);
 894                 }
 895         }
 896
 897         *empty = false;
 898         return (ret);
 899 }
 900
 901 /*
 902  * A helper function for monthcoll.  If a line matches
 903  * a month name, it returns (number of the month - 1),
 904  * while if there is no match, it just return -1.
 905  */
 906
 907 int
 908 bws_month_score(const struct bwstring *s0)
 909 {
 910
 911         if (MB_CUR_MAX == 1) {
 912                 const unsigned char *end, *s;
 913                 size_t len;
 914
 915                 s = s0->data.cstr;
 916                 end = s + s0->len;
 917
 918                 while (isblank(*s) && s < end)
 919                         ++s;
 920
 921                 len = strlen((const char*)s);
 922
 923                 for (int i = 11; i >= 0; --i) {
 924                         if (cmonths[i] &&
 925                             (s == (unsigned char*)strstr((const char*)s, (char*)(cmonths[i]))))
 926                                 return (i);
 927                 }
 928
 929         } else {
 930                 const wchar_t *end, *s;
 931                 size_t len;
 932
 933                 s = s0->data.wstr;
 934                 end = s + s0->len;
 935
 936                 while (iswblank(*s) && s < end)
 937                         ++s;
 938
 939                 len = wcslen(s);
 940
 941                 for (int i = 11; i >= 0; --i) {
 942                         if (wmonths[i] && (s == wcsstr(s, wmonths[i])))
 943                                 return (i);
 944                 }
 945         }
 946
 947         return (-1);
 948 }
 949
 950 /*
 951  * Rips out leading blanks (-b).
 952  */
 953 struct bwstring *
 954 ignore_leading_blanks(struct bwstring *str)
 955 {
 956
 957         if (MB_CUR_MAX == 1) {
 958                 unsigned char *dst, *end, *src;
 959
 960                 src = str->data.cstr;
 961                 dst = src;
 962                 end = src + str->len;
 963
 964                 while (src < end && isblank(*src))
 965                         ++src;
 966
 967                 if (src != dst) {
 968                         size_t newlen;
 969
 970                         newlen = BWSLEN(str) - (src - dst);
 971
 972                         while (src < end) {
 973                                 *dst = *src;
 974                                 ++dst;
 975                                 ++src;
 976                         }
 977                         bws_setlen(str, newlen);
 978                 }
 979         } else {
 980                 wchar_t *dst, *end, *src;
 981
 982                 src = str->data.wstr;
 983                 dst = src;
 984                 end = src + str->len;
 985
 986                 while (src < end && iswblank(*src))
 987                         ++src;
 988
 989                 if (src != dst) {
 990
 991                         size_t newlen = BWSLEN(str) - (src - dst);
 992
 993                         while (src < end) {
 994                                 *dst = *src;
 995                                 ++dst;
 996                                 ++src;
 997                         }
 998                         bws_setlen(str, newlen);
 999
1000                 }
1001         }
1002         return (str);
1003 }
1004
1005 /*
1006  * Rips out nonprinting characters (-i).
1007  */
1008 struct bwstring *
1009 ignore_nonprinting(struct bwstring *str)
1010 {
1011         size_t newlen = str->len;
1012
1013         if (MB_CUR_MAX == 1) {
1014                 unsigned char *dst, *end, *src;
1015                 unsigned char c;
1016
1017                 src = str->data.cstr;
1018                 dst = src;
1019                 end = src + str->len;
1020
1021                 while (src < end) {
1022                         c = *src;
1023                         if (isprint(c)) {
1024                                 *dst = c;
1025                                 ++dst;
1026                                 ++src;
1027                         } else {
1028                                 ++src;
1029                                 --newlen;
1030                         }
1031                 }
1032         } else {
1033                 wchar_t *dst, *end, *src;
1034                 wchar_t c;
1035
1036                 src = str->data.wstr;
1037                 dst = src;
1038                 end = src + str->len;
1039
1040                 while (src < end) {
1041                         c = *src;
1042                         if (iswprint(c)) {
1043                                 *dst = c;
1044                                 ++dst;
1045                                 ++src;
1046                         } else {
1047                                 ++src;
1048                                 --newlen;
1049                         }
1050                 }
1051         }
1052         bws_setlen(str, newlen);
1053
1054         return (str);
1055 }
1056
1057 /*
1058  * Rips out any characters that are not alphanumeric characters
1059  * nor blanks (-d).
1060  */
1061 struct bwstring *
1062 dictionary_order(struct bwstring *str)
1063 {
1064         size_t newlen = str->len;
1065
1066         if (MB_CUR_MAX == 1) {
1067                 unsigned char *dst, *end, *src;
1068                 unsigned char c;
1069
1070                 src = str->data.cstr;
1071                 dst = src;
1072                 end = src + str->len;
1073
1074                 while (src < end) {
1075                         c = *src;
1076                         if (isalnum(c) || isblank(c)) {
1077                                 *dst = c;
1078                                 ++dst;
1079                                 ++src;
1080                         } else {
1081                                 ++src;
1082                                 --newlen;
1083                         }
1084                 }
1085         } else {
1086                 wchar_t *dst, *end, *src;
1087                 wchar_t c;
1088
1089                 src = str->data.wstr;
1090                 dst = src;
1091                 end = src + str->len;
1092
1093                 while (src < end) {
1094                         c = *src;
1095                         if (iswalnum(c) || iswblank(c)) {
1096                                 *dst = c;
1097                                 ++dst;
1098                                 ++src;
1099                         } else {
1100                                 ++src;
1101                                 --newlen;
1102                         }
1103                 }
1104         }
1105         bws_setlen(str, newlen);
1106
1107         return (str);
1108 }
1109
1110 /*
1111  * Converts string to lower case(-f).
1112  */
1113 struct bwstring *
1114 ignore_case(struct bwstring *str)
1115 {
1116
1117         if (MB_CUR_MAX == 1) {
1118                 unsigned char *end, *s;
1119
1120                 s = str->data.cstr;
1121                 end = s + str->len;
1122
1123                 while (s < end) {
1124                         *s = toupper(*s);
1125                         ++s;
1126                 }
1127         } else {
1128                 wchar_t *end, *s;
1129
1130                 s = str->data.wstr;
1131                 end = s + str->len;
1132
1133                 while (s < end) {
1134                         *s = towupper(*s);
1135                         ++s;
1136                 }
1137         }
1138         return (str);
1139 }
1140
1141 void
1142 bws_disorder_warnx(struct bwstring *s, const char *fn, size_t pos)
1143 {
1144
1145         if (MB_CUR_MAX == 1)
1146                 warnx("%s:%zu: disorder: %s", fn, pos + 1, s->data.cstr);
1147         else
1148                 warnx("%s:%zu: disorder: %ls", fn, pos + 1, s->data.wstr);
1149 }