contrib/libucl/src/ucl_parser.c

   1 /* Copyright (c) 2013, Vsevolod Stakhov
   2  * All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions are met:
   6  *       * Redistributions of source code must retain the above copyright
   7  *         notice, this list of conditions and the following disclaimer.
   8  *       * Redistributions in binary form must reproduce the above copyright
   9  *         notice, this list of conditions and the following disclaimer in the
  10  *         documentation and/or other materials provided with the distribution.
  11  *
  12  * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
  13  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  14  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  15  * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
  16  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  17  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  18  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  19  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  20  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  21  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  22  */
  23
  24 #include "ucl.h"
  25 #include "ucl_internal.h"
  26 #include "ucl_chartable.h"
  27
  28 /**
  29  * @file ucl_parser.c
  30  * The implementation of ucl parser
  31  */
  32
  33 struct ucl_parser_saved_state {
  34         unsigned int line;
  35         unsigned int column;
  36         size_t remain;
  37         const unsigned char *pos;
  38 };
  39
  40 /**
  41  * Move up to len characters
  42  * @param parser
  43  * @param begin
  44  * @param len
  45  * @return new position in chunk
  46  */
  47 #define ucl_chunk_skipc(chunk, p)    do{                                        \
  48     if (*(p) == '\n') {                                                                         \
  49         (chunk)->line ++;                                                                       \
  50         (chunk)->column = 0;                                                            \
  51     }                                                                                                           \
  52     else (chunk)->column ++;                                                            \
  53     (p++);                                                                                                      \
  54     (chunk)->pos ++;                                                                            \
  55     (chunk)->remain --;                                                                         \
  56     } while (0)
  57
  58 static inline void
  59 ucl_set_err (struct ucl_parser *parser, int code, const char *str, UT_string **err)
  60 {
  61         const char *fmt_string, *filename;
  62         struct ucl_chunk *chunk = parser->chunks;
  63
  64         if (parser->cur_file) {
  65                 filename = parser->cur_file;
  66         }
  67         else {
  68                 filename = "<unknown>";
  69         }
  70
  71         if (chunk->pos < chunk->end) {
  72                 if (isgraph (*chunk->pos)) {
  73                         fmt_string = "error while parsing %s: "
  74                                         "line: %d, column: %d - '%s', character: '%c'";
  75                 }
  76                 else {
  77                         fmt_string = "error while parsing %s: "
  78                                         "line: %d, column: %d - '%s', character: '0x%02x'";
  79                 }
  80                 ucl_create_err (err, fmt_string,
  81                         filename, chunk->line, chunk->column,
  82                         str, *chunk->pos);
  83         }
  84         else {
  85                 ucl_create_err (err, "error while parsing %s: at the end of chunk: %s",
  86                         filename, str);
  87         }
  88
  89         parser->err_code = code;
  90 }
  91
  92 /**
  93  * Skip all comments from the current pos resolving nested and multiline comments
  94  * @param parser
  95  * @return
  96  */
  97 static bool
  98 ucl_skip_comments (struct ucl_parser *parser)
  99 {
 100         struct ucl_chunk *chunk = parser->chunks;
 101         const unsigned char *p;
 102         int comments_nested = 0;
 103         bool quoted = false;
 104
 105         p = chunk->pos;
 106
 107 start:
 108         if (chunk->remain > 0 && *p == '#') {
 109                 if (parser->state != UCL_STATE_SCOMMENT &&
 110                                 parser->state != UCL_STATE_MCOMMENT) {
 111                         while (p < chunk->end) {
 112                                 if (*p == '\n') {
 113                                         ucl_chunk_skipc (chunk, p);
 114                                         goto start;
 115                                 }
 116                                 ucl_chunk_skipc (chunk, p);
 117                         }
 118                 }
 119         }
 120         else if (chunk->remain >= 2 && *p == '/') {
 121                 if (p[1] == '*') {
 122                         ucl_chunk_skipc (chunk, p);
 123                         comments_nested ++;
 124                         ucl_chunk_skipc (chunk, p);
 125
 126                         while (p < chunk->end) {
 127                                 if (*p == '"' && *(p - 1) != '\\') {
 128                                         quoted = !quoted;
 129                                 }
 130
 131                                 if (!quoted) {
 132                                         if (*p == '*') {
 133                                                 ucl_chunk_skipc (chunk, p);
 134                                                 if (*p == '/') {
 135                                                         comments_nested --;
 136                                                         if (comments_nested == 0) {
 137                                                                 ucl_chunk_skipc (chunk, p);
 138                                                                 goto start;
 139                                                         }
 140                                                 }
 141                                                 ucl_chunk_skipc (chunk, p);
 142                                         }
 143                                         else if (p[0] == '/' && chunk->remain >= 2 && p[1] == '*') {
 144                                                 comments_nested ++;
 145                                                 ucl_chunk_skipc (chunk, p);
 146                                                 ucl_chunk_skipc (chunk, p);
 147                                                 continue;
 148                                         }
 149                                 }
 150                                 ucl_chunk_skipc (chunk, p);
 151                         }
 152                         if (comments_nested != 0) {
 153                                 ucl_set_err (parser, UCL_ENESTED,
 154                                                 "unfinished multiline comment", &parser->err);
 155                                 return false;
 156                         }
 157                 }
 158         }
 159
 160         return true;
 161 }
 162
 163 /**
 164  * Return multiplier for a character
 165  * @param c multiplier character
 166  * @param is_bytes if true use 1024 multiplier
 167  * @return multiplier
 168  */
 169 static inline unsigned long
 170 ucl_lex_num_multiplier (const unsigned char c, bool is_bytes) {
 171         const struct {
 172                 char c;
 173                 long mult_normal;
 174                 long mult_bytes;
 175         } multipliers[] = {
 176                         {'m', 1000 * 1000, 1024 * 1024},
 177                         {'k', 1000, 1024},
 178                         {'g', 1000 * 1000 * 1000, 1024 * 1024 * 1024}
 179         };
 180         int i;
 181
 182         for (i = 0; i < 3; i ++) {
 183                 if (tolower (c) == multipliers[i].c) {
 184                         if (is_bytes) {
 185                                 return multipliers[i].mult_bytes;
 186                         }
 187                         return multipliers[i].mult_normal;
 188                 }
 189         }
 190
 191         return 1;
 192 }
 193
 194
 195 /**
 196  * Return multiplier for time scaling
 197  * @param c
 198  * @return
 199  */
 200 static inline double
 201 ucl_lex_time_multiplier (const unsigned char c) {
 202         const struct {
 203                 char c;
 204                 double mult;
 205         } multipliers[] = {
 206                         {'m', 60},
 207                         {'h', 60 * 60},
 208                         {'d', 60 * 60 * 24},
 209                         {'w', 60 * 60 * 24 * 7},
 210                         {'y', 60 * 60 * 24 * 7 * 365}
 211         };
 212         int i;
 213
 214         for (i = 0; i < 5; i ++) {
 215                 if (tolower (c) == multipliers[i].c) {
 216                         return multipliers[i].mult;
 217                 }
 218         }
 219
 220         return 1;
 221 }
 222
 223 /**
 224  * Return true if a character is a end of an atom
 225  * @param c
 226  * @return
 227  */
 228 static inline bool
 229 ucl_lex_is_atom_end (const unsigned char c)
 230 {
 231         return ucl_test_character (c, UCL_CHARACTER_VALUE_END);
 232 }
 233
 234 static inline bool
 235 ucl_lex_is_comment (const unsigned char c1, const unsigned char c2)
 236 {
 237         if (c1 == '/') {
 238                 if (c2 == '*') {
 239                         return true;
 240                 }
 241         }
 242         else if (c1 == '#') {
 243                 return true;
 244         }
 245         return false;
 246 }
 247
 248 /**
 249  * Check variable found
 250  * @param parser
 251  * @param ptr
 252  * @param remain
 253  * @param out_len
 254  * @param strict
 255  * @param found
 256  * @return
 257  */
 258 static inline const char *
 259 ucl_check_variable_safe (struct ucl_parser *parser, const char *ptr, size_t remain,
 260                 size_t *out_len, bool strict, bool *found)
 261 {
 262         struct ucl_variable *var;
 263         unsigned char *dst;
 264         size_t dstlen;
 265         bool need_free = false;
 266
 267         LL_FOREACH (parser->variables, var) {
 268                 if (strict) {
 269                         if (remain == var->var_len) {
 270                                 if (memcmp (ptr, var->var, var->var_len) == 0) {
 271                                         *out_len += var->value_len;
 272                                         *found = true;
 273                                         return (ptr + var->var_len);
 274                                 }
 275                         }
 276                 }
 277                 else {
 278                         if (remain >= var->var_len) {
 279                                 if (memcmp (ptr, var->var, var->var_len) == 0) {
 280                                         *out_len += var->value_len;
 281                                         *found = true;
 282                                         return (ptr + var->var_len);
 283                                 }
 284                         }
 285                 }
 286         }
 287
 288         /* XXX: can only handle ${VAR} */
 289         if (!(*found) && parser->var_handler != NULL && strict) {
 290                 /* Call generic handler */
 291                 if (parser->var_handler (ptr, remain, &dst, &dstlen, &need_free,
 292                                 parser->var_data)) {
 293                         *found = true;
 294                         if (need_free) {
 295                                 free (dst);
 296                         }
 297                         return (ptr + remain);
 298                 }
 299         }
 300
 301         return ptr;
 302 }
 303
 304 /**
 305  * Check for a variable in a given string
 306  * @param parser
 307  * @param ptr
 308  * @param remain
 309  * @param out_len
 310  * @param vars_found
 311  * @return
 312  */
 313 static const char *
 314 ucl_check_variable (struct ucl_parser *parser, const char *ptr,
 315                 size_t remain, size_t *out_len, bool *vars_found)
 316 {
 317         const char *p, *end, *ret = ptr;
 318         bool found = false;
 319
 320         if (*ptr == '{') {
 321                 /* We need to match the variable enclosed in braces */
 322                 p = ptr + 1;
 323                 end = ptr + remain;
 324                 while (p < end) {
 325                         if (*p == '}') {
 326                                 ret = ucl_check_variable_safe (parser, ptr + 1, p - ptr - 1,
 327                                                 out_len, true, &found);
 328                                 if (found) {
 329                                         /* {} must be excluded actually */
 330                                         ret ++;
 331                                         if (!*vars_found) {
 332                                                 *vars_found = true;
 333                                         }
 334                                 }
 335                                 else {
 336                                         *out_len += 2;
 337                                 }
 338                                 break;
 339                         }
 340                         p ++;
 341                 }
 342         }
 343         else if (*ptr != '$') {
 344                 /* Not count escaped dollar sign */
 345                 ret = ucl_check_variable_safe (parser, ptr, remain, out_len, false, &found);
 346                 if (found && !*vars_found) {
 347                         *vars_found = true;
 348                 }
 349                 if (!found) {
 350                         (*out_len) ++;
 351                 }
 352         }
 353         else {
 354                 ret ++;
 355                 (*out_len) ++;
 356         }
 357
 358         return ret;
 359 }
 360
 361 /**
 362  * Expand a single variable
 363  * @param parser
 364  * @param ptr
 365  * @param remain
 366  * @param dest
 367  * @return
 368  */
 369 static const char *
 370 ucl_expand_single_variable (struct ucl_parser *parser, const char *ptr,
 371                 size_t remain, unsigned char **dest)
 372 {
 373         unsigned char *d = *dest, *dst;
 374         const char *p = ptr + 1, *ret;
 375         struct ucl_variable *var;
 376         size_t dstlen;
 377         bool need_free = false;
 378         bool found = false;
 379         bool strict = false;
 380
 381         ret = ptr + 1;
 382         remain --;
 383
 384         if (*p == '$') {
 385                 *d++ = *p++;
 386                 *dest = d;
 387                 return p;
 388         }
 389         else if (*p == '{') {
 390                 p ++;
 391                 strict = true;
 392                 ret += 2;
 393                 remain -= 2;
 394         }
 395
 396         LL_FOREACH (parser->variables, var) {
 397                 if (remain >= var->var_len) {
 398                         if (memcmp (p, var->var, var->var_len) == 0) {
 399                                 memcpy (d, var->value, var->value_len);
 400                                 ret += var->var_len;
 401                                 d += var->value_len;
 402                                 found = true;
 403                                 break;
 404                         }
 405                 }
 406         }
 407         if (!found) {
 408                 if (strict && parser->var_handler != NULL) {
 409                         if (parser->var_handler (ptr, remain, &dst, &dstlen, &need_free,
 410                                                         parser->var_data)) {
 411                                 memcpy (d, dst, dstlen);
 412                                 ret += dstlen;
 413                                 d += remain;
 414                                 found = true;
 415                         }
 416                 }
 417
 418                 /* Leave variable as is */
 419                 if (!found) {
 420                         if (strict) {
 421                                 /* Copy '${' */
 422                                 memcpy (d, ptr, 2);
 423                                 d += 2;
 424                                 ret --;
 425                         }
 426                         else {
 427                                 memcpy (d, ptr, 1);
 428                                 d ++;
 429                         }
 430                 }
 431         }
 432
 433         *dest = d;
 434         return ret;
 435 }
 436
 437 /**
 438  * Expand variables in string
 439  * @param parser
 440  * @param dst
 441  * @param src
 442  * @param in_len
 443  * @return
 444  */
 445 static ssize_t
 446 ucl_expand_variable (struct ucl_parser *parser, unsigned char **dst,
 447                 const char *src, size_t in_len)
 448 {
 449         const char *p, *end = src + in_len;
 450         unsigned char *d;
 451         size_t out_len = 0;
 452         bool vars_found = false;
 453
 454         p = src;
 455         while (p != end) {
 456                 if (*p == '$') {
 457                         p = ucl_check_variable (parser, p + 1, end - p - 1, &out_len, &vars_found);
 458                 }
 459                 else {
 460                         p ++;
 461                         out_len ++;
 462                 }
 463         }
 464
 465         if (!vars_found) {
 466                 /* Trivial case */
 467                 *dst = NULL;
 468                 return in_len;
 469         }
 470
 471         *dst = UCL_ALLOC (out_len + 1);
 472         if (*dst == NULL) {
 473                 return in_len;
 474         }
 475
 476         d = *dst;
 477         p = src;
 478         while (p != end) {
 479                 if (*p == '$') {
 480                         p = ucl_expand_single_variable (parser, p, end - p, &d);
 481                 }
 482                 else {
 483                         *d++ = *p++;
 484                 }
 485         }
 486
 487         *d = '\0';
 488
 489         return out_len;
 490 }
 491
 492 /**
 493  * Store or copy pointer to the trash stack
 494  * @param parser parser object
 495  * @param src src string
 496  * @param dst destination buffer (trash stack pointer)
 497  * @param dst_const const destination pointer (e.g. value of object)
 498  * @param in_len input length
 499  * @param need_unescape need to unescape source (and copy it)
 500  * @param need_lowercase need to lowercase value (and copy)
 501  * @param need_expand need to expand variables (and copy as well)
 502  * @return output length (excluding \0 symbol)
 503  */
 504 static inline ssize_t
 505 ucl_copy_or_store_ptr (struct ucl_parser *parser,
 506                 const unsigned char *src, unsigned char **dst,
 507                 const char **dst_const, size_t in_len,
 508                 bool need_unescape, bool need_lowercase, bool need_expand)
 509 {
 510         ssize_t ret = -1, tret;
 511         unsigned char *tmp;
 512
 513         if (need_unescape || need_lowercase ||
 514                         (need_expand && parser->variables != NULL) ||
 515                         !(parser->flags & UCL_PARSER_ZEROCOPY)) {
 516                 /* Copy string */
 517                 *dst = UCL_ALLOC (in_len + 1);
 518                 if (*dst == NULL) {
 519                         ucl_set_err (parser, UCL_EINTERNAL, "cannot allocate memory for a string",
 520                                         &parser->err);
 521                         return false;
 522                 }
 523                 if (need_lowercase) {
 524                         ret = ucl_strlcpy_tolower (*dst, src, in_len + 1);
 525                 }
 526                 else {
 527                         ret = ucl_strlcpy_unsafe (*dst, src, in_len + 1);
 528                 }
 529
 530                 if (need_unescape) {
 531                         ret = ucl_unescape_json_string (*dst, ret);
 532                 }
 533                 if (need_expand) {
 534                         tmp = *dst;
 535                         tret = ret;
 536                         ret = ucl_expand_variable (parser, dst, tmp, ret);
 537                         if (*dst == NULL) {
 538                                 /* Nothing to expand */
 539                                 *dst = tmp;
 540                                 ret = tret;
 541                         }
 542                         else {
 543                                 /* Free unexpanded value */
 544                                 UCL_FREE (in_len + 1, tmp);
 545                         }
 546                 }
 547                 *dst_const = *dst;
 548         }
 549         else {
 550                 *dst_const = src;
 551                 ret = in_len;
 552         }
 553
 554         return ret;
 555 }
 556
 557 /**
 558  * Create and append an object at the specified level
 559  * @param parser
 560  * @param is_array
 561  * @param level
 562  * @return
 563  */
 564 static inline ucl_object_t *
 565 ucl_parser_add_container (ucl_object_t *obj, struct ucl_parser *parser,
 566                 bool is_array, int level)
 567 {
 568         struct ucl_stack *st;
 569
 570         if (!is_array) {
 571                 if (obj == NULL) {
 572                         obj = ucl_object_new_full (UCL_OBJECT, parser->chunks->priority);
 573                 }
 574                 else {
 575                         obj->type = UCL_OBJECT;
 576                 }
 577                 if (obj->value.ov == NULL) {
 578                         obj->value.ov = ucl_hash_create (parser->flags & UCL_PARSER_KEY_LOWERCASE);
 579                 }
 580                 parser->state = UCL_STATE_KEY;
 581         }
 582         else {
 583                 if (obj == NULL) {
 584                         obj = ucl_object_new_full (UCL_ARRAY, parser->chunks->priority);
 585                 }
 586                 else {
 587                         obj->type = UCL_ARRAY;
 588                 }
 589                 parser->state = UCL_STATE_VALUE;
 590         }
 591
 592         st = UCL_ALLOC (sizeof (struct ucl_stack));
 593         if (st == NULL) {
 594                 ucl_set_err (parser, UCL_EINTERNAL, "cannot allocate memory for an object",
 595                                 &parser->err);
 596                 ucl_object_unref (obj);
 597                 return NULL;
 598         }
 599         st->obj = obj;
 600         st->level = level;
 601         LL_PREPEND (parser->stack, st);
 602         parser->cur_obj = obj;
 603
 604         return obj;
 605 }
 606
 607 int
 608 ucl_maybe_parse_number (ucl_object_t *obj,
 609                 const char *start, const char *end, const char **pos,
 610                 bool allow_double, bool number_bytes, bool allow_time)
 611 {
 612         const char *p = start, *c = start;
 613         char *endptr;
 614         bool got_dot = false, got_exp = false, need_double = false,
 615                         is_time = false, valid_start = false, is_hex = false,
 616                         is_neg = false;
 617         double dv = 0;
 618         int64_t lv = 0;
 619
 620         if (*p == '-') {
 621                 is_neg = true;
 622                 c ++;
 623                 p ++;
 624         }
 625         while (p < end) {
 626                 if (is_hex && isxdigit (*p)) {
 627                         p ++;
 628                 }
 629                 else if (isdigit (*p)) {
 630                         valid_start = true;
 631                         p ++;
 632                 }
 633                 else if (!is_hex && (*p == 'x' || *p == 'X')) {
 634                         is_hex = true;
 635                         allow_double = false;
 636                         c = p + 1;
 637                 }
 638                 else if (allow_double) {
 639                         if (p == c) {
 640                                 /* Empty digits sequence, not a number */
 641                                 *pos = start;
 642                                 return EINVAL;
 643                         }
 644                         else if (*p == '.') {
 645                                 if (got_dot) {
 646                                         /* Double dots, not a number */
 647                                         *pos = start;
 648                                         return EINVAL;
 649                                 }
 650                                 else {
 651                                         got_dot = true;
 652                                         need_double = true;
 653                                         p ++;
 654                                 }
 655                         }
 656                         else if (*p == 'e' || *p == 'E') {
 657                                 if (got_exp) {
 658                                         /* Double exp, not a number */
 659                                         *pos = start;
 660                                         return EINVAL;
 661                                 }
 662                                 else {
 663                                         got_exp = true;
 664                                         need_double = true;
 665                                         p ++;
 666                                         if (p >= end) {
 667                                                 *pos = start;
 668                                                 return EINVAL;
 669                                         }
 670                                         if (!isdigit (*p) && *p != '+' && *p != '-') {
 671                                                 /* Wrong exponent sign */
 672                                                 *pos = start;
 673                                                 return EINVAL;
 674                                         }
 675                                         else {
 676                                                 p ++;
 677                                         }
 678                                 }
 679                         }
 680                         else {
 681                                 /* Got the end of the number, need to check */
 682                                 break;
 683                         }
 684                 }
 685                 else {
 686                         break;
 687                 }
 688         }
 689
 690         if (!valid_start) {
 691                 *pos = start;
 692                 return EINVAL;
 693         }
 694
 695         errno = 0;
 696         if (need_double) {
 697                 dv = strtod (c, &endptr);
 698         }
 699         else {
 700                 if (is_hex) {
 701                         lv = strtoimax (c, &endptr, 16);
 702                 }
 703                 else {
 704                         lv = strtoimax (c, &endptr, 10);
 705                 }
 706         }
 707         if (errno == ERANGE) {
 708                 *pos = start;
 709                 return ERANGE;
 710         }
 711
 712         /* Now check endptr */
 713         if (endptr == NULL || ucl_lex_is_atom_end (*endptr) || *endptr == '\0') {
 714                 p = endptr;
 715                 goto set_obj;
 716         }
 717
 718         if (endptr < end && endptr != start) {
 719                 p = endptr;
 720                 switch (*p) {
 721                 case 'm':
 722                 case 'M':
 723                 case 'g':
 724                 case 'G':
 725                 case 'k':
 726                 case 'K':
 727                         if (end - p >= 2) {
 728                                 if (p[1] == 's' || p[1] == 'S') {
 729                                         /* Milliseconds */
 730                                         if (!need_double) {
 731                                                 need_double = true;
 732                                                 dv = lv;
 733                                         }
 734                                         is_time = true;
 735                                         if (p[0] == 'm' || p[0] == 'M') {
 736                                                 dv /= 1000.;
 737                                         }
 738                                         else {
 739                                                 dv *= ucl_lex_num_multiplier (*p, false);
 740                                         }
 741                                         p += 2;
 742                                         goto set_obj;
 743                                 }
 744                                 else if (number_bytes || (p[1] == 'b' || p[1] == 'B')) {
 745                                         /* Bytes */
 746                                         if (need_double) {
 747                                                 need_double = false;
 748                                                 lv = dv;
 749                                         }
 750                                         lv *= ucl_lex_num_multiplier (*p, true);
 751                                         p += 2;
 752                                         goto set_obj;
 753                                 }
 754                                 else if (ucl_lex_is_atom_end (p[1])) {
 755                                         if (need_double) {
 756                                                 dv *= ucl_lex_num_multiplier (*p, false);
 757                                         }
 758                                         else {
 759                                                 lv *= ucl_lex_num_multiplier (*p, number_bytes);
 760                                         }
 761                                         p ++;
 762                                         goto set_obj;
 763                                 }
 764                                 else if (allow_time && end - p >= 3) {
 765                                         if (tolower (p[0]) == 'm' &&
 766                                                         tolower (p[1]) == 'i' &&
 767                                                         tolower (p[2]) == 'n') {
 768                                                 /* Minutes */
 769                                                 if (!need_double) {
 770                                                         need_double = true;
 771                                                         dv = lv;
 772                                                 }
 773                                                 is_time = true;
 774                                                 dv *= 60.;
 775                                                 p += 3;
 776                                                 goto set_obj;
 777                                         }
 778                                 }
 779                         }
 780                         else {
 781                                 if (need_double) {
 782                                         dv *= ucl_lex_num_multiplier (*p, false);
 783                                 }
 784                                 else {
 785                                         lv *= ucl_lex_num_multiplier (*p, number_bytes);
 786                                 }
 787                                 p ++;
 788                                 goto set_obj;
 789                         }
 790                         break;
 791                 case 'S':
 792                 case 's':
 793                         if (allow_time &&
 794                                         (p == end - 1 || ucl_lex_is_atom_end (p[1]))) {
 795                                 if (!need_double) {
 796                                         need_double = true;
 797                                         dv = lv;
 798                                 }
 799                                 p ++;
 800                                 is_time = true;
 801                                 goto set_obj;
 802                         }
 803                         break;
 804                 case 'h':
 805                 case 'H':
 806                 case 'd':
 807                 case 'D':
 808                 case 'w':
 809                 case 'W':
 810                 case 'Y':
 811                 case 'y':
 812                         if (allow_time &&
 813                                         (p == end - 1 || ucl_lex_is_atom_end (p[1]))) {
 814                                 if (!need_double) {
 815                                         need_double = true;
 816                                         dv = lv;
 817                                 }
 818                                 is_time = true;
 819                                 dv *= ucl_lex_time_multiplier (*p);
 820                                 p ++;
 821                                 goto set_obj;
 822                         }
 823                         break;
 824                 case '\t':
 825                 case ' ':
 826                         while (p < end && ucl_test_character(*p, UCL_CHARACTER_WHITESPACE)) {
 827                                 p++;
 828                         }
 829                         if (ucl_lex_is_atom_end(*p))
 830                                 goto set_obj;
 831                         break;
 832                 }
 833         }
 834         else if (endptr == end) {
 835                 /* Just a number at the end of chunk */
 836                 p = endptr;
 837                 goto set_obj;
 838         }
 839
 840         *pos = c;
 841         return EINVAL;
 842
 843 set_obj:
 844         if (obj != NULL) {
 845                 if (allow_double && (need_double || is_time)) {
 846                         if (!is_time) {
 847                                 obj->type = UCL_FLOAT;
 848                         }
 849                         else {
 850                                 obj->type = UCL_TIME;
 851                         }
 852                         obj->value.dv = is_neg ? (-dv) : dv;
 853                 }
 854                 else {
 855                         obj->type = UCL_INT;
 856                         obj->value.iv = is_neg ? (-lv) : lv;
 857                 }
 858         }
 859         *pos = p;
 860         return 0;
 861 }
 862
 863 /**
 864  * Parse possible number
 865  * @param parser
 866  * @param chunk
 867  * @param obj
 868  * @return true if a number has been parsed
 869  */
 870 static bool
 871 ucl_lex_number (struct ucl_parser *parser,
 872                 struct ucl_chunk *chunk, ucl_object_t *obj)
 873 {
 874         const unsigned char *pos;
 875         int ret;
 876
 877         ret = ucl_maybe_parse_number (obj, chunk->pos, chunk->end, (const char **)&pos,
 878                         true, false, ((parser->flags & UCL_PARSER_NO_TIME) == 0));
 879
 880         if (ret == 0) {
 881                 chunk->remain -= pos - chunk->pos;
 882                 chunk->column += pos - chunk->pos;
 883                 chunk->pos = pos;
 884                 return true;
 885         }
 886         else if (ret == ERANGE) {
 887                 ucl_set_err (parser, UCL_ESYNTAX, "numeric value out of range",
 888                                 &parser->err);
 889         }
 890
 891         return false;
 892 }
 893
 894 /**
 895  * Parse quoted string with possible escapes
 896  * @param parser
 897  * @param chunk
 898  * @param need_unescape
 899  * @param ucl_escape
 900  * @param var_expand
 901  * @return true if a string has been parsed
 902  */
 903 static bool
 904 ucl_lex_json_string (struct ucl_parser *parser,
 905                 struct ucl_chunk *chunk, bool *need_unescape, bool *ucl_escape, bool *var_expand)
 906 {
 907         const unsigned char *p = chunk->pos;
 908         unsigned char c;
 909         int i;
 910
 911         while (p < chunk->end) {
 912                 c = *p;
 913                 if (c < 0x1F) {
 914                         /* Unmasked control character */
 915                         if (c == '\n') {
 916                                 ucl_set_err (parser, UCL_ESYNTAX, "unexpected newline",
 917                                                 &parser->err);
 918                         }
 919                         else {
 920                                 ucl_set_err (parser, UCL_ESYNTAX, "unexpected control character",
 921                                                 &parser->err);
 922                         }
 923                         return false;
 924                 }
 925                 else if (c == '\\') {
 926                         ucl_chunk_skipc (chunk, p);
 927                         c = *p;
 928                         if (p >= chunk->end) {
 929                                 ucl_set_err (parser, UCL_ESYNTAX, "unfinished escape character",
 930                                                 &parser->err);
 931                                 return false;
 932                         }
 933                         else if (ucl_test_character (c, UCL_CHARACTER_ESCAPE)) {
 934                                 if (c == 'u') {
 935                                         ucl_chunk_skipc (chunk, p);
 936                                         for (i = 0; i < 4 && p < chunk->end; i ++) {
 937                                                 if (!isxdigit (*p)) {
 938                                                         ucl_set_err (parser, UCL_ESYNTAX, "invalid utf escape",
 939                                                                         &parser->err);
 940                                                         return false;
 941                                                 }
 942                                                 ucl_chunk_skipc (chunk, p);
 943                                         }
 944                                         if (p >= chunk->end) {
 945                                                 ucl_set_err (parser, UCL_ESYNTAX, "unfinished escape character",
 946                                                                 &parser->err);
 947                                                 return false;
 948                                         }
 949                                 }
 950                                 else {
 951                                         ucl_chunk_skipc (chunk, p);
 952                                 }
 953                         }
 954                         *need_unescape = true;
 955                         *ucl_escape = true;
 956                         continue;
 957                 }
 958                 else if (c == '"') {
 959                         ucl_chunk_skipc (chunk, p);
 960                         return true;
 961                 }
 962                 else if (ucl_test_character (c, UCL_CHARACTER_UCL_UNSAFE)) {
 963                         *ucl_escape = true;
 964                 }
 965                 else if (c == '$') {
 966                         *var_expand = true;
 967                 }
 968                 ucl_chunk_skipc (chunk, p);
 969         }
 970
 971         ucl_set_err (parser, UCL_ESYNTAX, "no quote at the end of json string",
 972                         &parser->err);
 973         return false;
 974 }
 975
 976 static void
 977 ucl_parser_append_elt (struct ucl_parser *parser, ucl_hash_t *cont,
 978                 ucl_object_t *top,
 979                 ucl_object_t *elt)
 980 {
 981         ucl_object_t *nobj;
 982
 983         if ((parser->flags & UCL_PARSER_NO_IMPLICIT_ARRAYS) == 0) {
 984                 /* Implicit array */
 985                 top->flags |= UCL_OBJECT_MULTIVALUE;
 986                 DL_APPEND (top, elt);
 987                 parser->stack->obj->len ++;
 988         }
 989         else {
 990                 if ((top->flags & UCL_OBJECT_MULTIVALUE) != 0) {
 991                         /* Just add to the explicit array */
 992                         ucl_array_append (top, elt);
 993                 }
 994                 else {
 995                         /* Convert to an array */
 996                         nobj = ucl_object_typed_new (UCL_ARRAY);
 997                         nobj->key = top->key;
 998                         nobj->keylen = top->keylen;
 999                         nobj->flags |= UCL_OBJECT_MULTIVALUE;
1000                         ucl_array_append (nobj, top);
1001                         ucl_array_append (nobj, elt);
1002                         ucl_hash_replace (cont, top, nobj);
1003                 }
1004         }
1005 }
1006
1007 bool
1008 ucl_parser_process_object_element (struct ucl_parser *parser, ucl_object_t *nobj)
1009 {
1010         ucl_hash_t *container;
1011         ucl_object_t *tobj;
1012
1013         container = parser->stack->obj->value.ov;
1014
1015         tobj = __DECONST (ucl_object_t *, ucl_hash_search_obj (container, nobj));
1016         if (tobj == NULL) {
1017                 container = ucl_hash_insert_object (container, nobj,
1018                                 parser->flags & UCL_PARSER_KEY_LOWERCASE);
1019                 nobj->prev = nobj;
1020                 nobj->next = NULL;
1021                 parser->stack->obj->len ++;
1022         }
1023         else {
1024                 unsigned priold = ucl_object_get_priority (tobj),
1025                                 prinew = ucl_object_get_priority (nobj);
1026                 switch (parser->chunks->strategy) {
1027
1028                 case UCL_DUPLICATE_APPEND:
1029                         /*
1030                          * The logic here is the following:
1031                          *
1032                          * - if we have two objects with the same priority, then we form an
1033                          * implicit or explicit array
1034                          * - if a new object has bigger priority, then we overwrite an old one
1035                          * - if a new object has lower priority, then we ignore it
1036                          */
1037
1038
1039                         /* Special case for inherited objects */
1040                         if (tobj->flags & UCL_OBJECT_INHERITED) {
1041                                 prinew = priold + 1;
1042                         }
1043
1044                         if (priold == prinew) {
1045                                 ucl_parser_append_elt (parser, container, tobj, nobj);
1046                         }
1047                         else if (priold > prinew) {
1048                                 /*
1049                                  * We add this new object to a list of trash objects just to ensure
1050                                  * that it won't come to any real object
1051                                  * XXX: rather inefficient approach
1052                                  */
1053                                 DL_APPEND (parser->trash_objs, nobj);
1054                         }
1055                         else {
1056                                 ucl_hash_replace (container, tobj, nobj);
1057                                 ucl_object_unref (tobj);
1058                         }
1059
1060                         break;
1061
1062                 case UCL_DUPLICATE_REWRITE:
1063                         /* We just rewrite old values regardless of priority */
1064                         ucl_hash_replace (container, tobj, nobj);
1065                         ucl_object_unref (tobj);
1066
1067                         break;
1068
1069                 case UCL_DUPLICATE_ERROR:
1070                         ucl_create_err (&parser->err, "error while parsing %s: "
1071                                         "line: %d, column: %d: duplicate element for key '%s' "
1072                                         "has been found",
1073                                         parser->cur_file ? parser->cur_file : "<unknown>",
1074                                         parser->chunks->line, parser->chunks->column, nobj->key);
1075                         return false;
1076
1077                 case UCL_DUPLICATE_MERGE:
1078                         /*
1079                          * Here we do have some old object so we just push it on top of objects stack
1080                          */
1081                         if (tobj->type == UCL_OBJECT || tobj->type == UCL_ARRAY) {
1082                                 ucl_object_unref (nobj);
1083                                 nobj = tobj;
1084                         }
1085                         else {
1086                                 /* For other types we create implicit array as usual */
1087                                 ucl_parser_append_elt (parser, container, tobj, nobj);
1088                         }
1089                         break;
1090                 }
1091         }
1092
1093         parser->stack->obj->value.ov = container;
1094         parser->cur_obj = nobj;
1095
1096         return true;
1097 }
1098
1099 /**
1100  * Parse a key in an object
1101  * @param parser
1102  * @param chunk
1103  * @param next_key
1104  * @param end_of_object
1105  * @return true if a key has been parsed
1106  */
1107 static bool
1108 ucl_parse_key (struct ucl_parser *parser, struct ucl_chunk *chunk,
1109                 bool *next_key, bool *end_of_object)
1110 {
1111         const unsigned char *p, *c = NULL, *end, *t;
1112         const char *key = NULL;
1113         bool got_quote = false, got_eq = false, got_semicolon = false,
1114                         need_unescape = false, ucl_escape = false, var_expand = false,
1115                         got_content = false, got_sep = false;
1116         ucl_object_t *nobj;
1117         ssize_t keylen;
1118
1119         p = chunk->pos;
1120
1121         if (*p == '.') {
1122                 /* It is macro actually */
1123                 ucl_chunk_skipc (chunk, p);
1124                 parser->prev_state = parser->state;
1125                 parser->state = UCL_STATE_MACRO_NAME;
1126                 *end_of_object = false;
1127                 return true;
1128         }
1129         while (p < chunk->end) {
1130                 /*
1131                  * A key must start with alpha, number, '/' or '_' and end with space character
1132                  */
1133                 if (c == NULL) {
1134                         if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) {
1135                                 if (!ucl_skip_comments (parser)) {
1136                                         return false;
1137                                 }
1138                                 p = chunk->pos;
1139                         }
1140                         else if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
1141                                 ucl_chunk_skipc (chunk, p);
1142                         }
1143                         else if (ucl_test_character (*p, UCL_CHARACTER_KEY_START)) {
1144                                 /* The first symbol */
1145                                 c = p;
1146                                 ucl_chunk_skipc (chunk, p);
1147                                 got_content = true;
1148                         }
1149                         else if (*p == '"') {
1150                                 /* JSON style key */
1151                                 c = p + 1;
1152                                 got_quote = true;
1153                                 got_content = true;
1154                                 ucl_chunk_skipc (chunk, p);
1155                         }
1156                         else if (*p == '}') {
1157                                 /* We have actually end of an object */
1158                                 *end_of_object = true;
1159                                 return true;
1160                         }
1161                         else if (*p == '.') {
1162                                 ucl_chunk_skipc (chunk, p);
1163                                 parser->prev_state = parser->state;
1164                                 parser->state = UCL_STATE_MACRO_NAME;
1165                                 return true;
1166                         }
1167                         else {
1168                                 /* Invalid identifier */
1169                                 ucl_set_err (parser, UCL_ESYNTAX, "key must begin with a letter",
1170                                                 &parser->err);
1171                                 return false;
1172                         }
1173                 }
1174                 else {
1175                         /* Parse the body of a key */
1176                         if (!got_quote) {
1177                                 if (ucl_test_character (*p, UCL_CHARACTER_KEY)) {
1178                                         got_content = true;
1179                                         ucl_chunk_skipc (chunk, p);
1180                                 }
1181                                 else if (ucl_test_character (*p, UCL_CHARACTER_KEY_SEP)) {
1182                                         end = p;
1183                                         break;
1184                                 }
1185                                 else {
1186                                         ucl_set_err (parser, UCL_ESYNTAX, "invalid character in a key",
1187                                                         &parser->err);
1188                                         return false;
1189                                 }
1190                         }
1191                         else {
1192                                 /* We need to parse json like quoted string */
1193                                 if (!ucl_lex_json_string (parser, chunk, &need_unescape, &ucl_escape, &var_expand)) {
1194                                         return false;
1195                                 }
1196                                 /* Always escape keys obtained via json */
1197                                 end = chunk->pos - 1;
1198                                 p = chunk->pos;
1199                                 break;
1200                         }
1201                 }
1202         }
1203
1204         if (p >= chunk->end && got_content) {
1205                 ucl_set_err (parser, UCL_ESYNTAX, "unfinished key", &parser->err);
1206                 return false;
1207         }
1208         else if (!got_content) {
1209                 return true;
1210         }
1211         *end_of_object = false;
1212         /* We are now at the end of the key, need to parse the rest */
1213         while (p < chunk->end) {
1214                 if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE)) {
1215                         ucl_chunk_skipc (chunk, p);
1216                 }
1217                 else if (*p == '=') {
1218                         if (!got_eq && !got_semicolon) {
1219                                 ucl_chunk_skipc (chunk, p);
1220                                 got_eq = true;
1221                         }
1222                         else {
1223                                 ucl_set_err (parser, UCL_ESYNTAX, "unexpected '=' character",
1224                                                 &parser->err);
1225                                 return false;
1226                         }
1227                 }
1228                 else if (*p == ':') {
1229                         if (!got_eq && !got_semicolon) {
1230                                 ucl_chunk_skipc (chunk, p);
1231                                 got_semicolon = true;
1232                         }
1233                         else {
1234                                 ucl_set_err (parser, UCL_ESYNTAX, "unexpected ':' character",
1235                                                 &parser->err);
1236                                 return false;
1237                         }
1238                 }
1239                 else if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) {
1240                         /* Check for comment */
1241                         if (!ucl_skip_comments (parser)) {
1242                                 return false;
1243                         }
1244                         p = chunk->pos;
1245                 }
1246                 else {
1247                         /* Start value */
1248                         break;
1249                 }
1250         }
1251
1252         if (p >= chunk->end && got_content) {
1253                 ucl_set_err (parser, UCL_ESYNTAX, "unfinished key", &parser->err);
1254                 return false;
1255         }
1256
1257         got_sep = got_semicolon || got_eq;
1258
1259         if (!got_sep) {
1260                 /*
1261                  * Maybe we have more keys nested, so search for termination character.
1262                  * Possible choices:
1263                  * 1) key1 key2 ... keyN [:=] value <- we treat that as error
1264                  * 2) key1 ... keyN {} or [] <- we treat that as nested objects
1265                  * 3) key1 value[;,\n] <- we treat that as linear object
1266                  */
1267                 t = p;
1268                 *next_key = false;
1269                 while (ucl_test_character (*t, UCL_CHARACTER_WHITESPACE)) {
1270                         t ++;
1271                 }
1272                 /* Check first non-space character after a key */
1273                 if (*t != '{' && *t != '[') {
1274                         while (t < chunk->end) {
1275                                 if (*t == ',' || *t == ';' || *t == '\n' || *t == '\r') {
1276                                         break;
1277                                 }
1278                                 else if (*t == '{' || *t == '[') {
1279                                         *next_key = true;
1280                                         break;
1281                                 }
1282                                 t ++;
1283                         }
1284                 }
1285         }
1286
1287         /* Create a new object */
1288         nobj = ucl_object_new_full (UCL_NULL, parser->chunks->priority);
1289         keylen = ucl_copy_or_store_ptr (parser, c, &nobj->trash_stack[UCL_TRASH_KEY],
1290                         &key, end - c, need_unescape, parser->flags & UCL_PARSER_KEY_LOWERCASE, false);
1291         if (keylen == -1) {
1292                 ucl_object_unref (nobj);
1293                 return false;
1294         }
1295         else if (keylen == 0) {
1296                 ucl_set_err (parser, UCL_ESYNTAX, "empty keys are not allowed", &parser->err);
1297                 ucl_object_unref (nobj);
1298                 return false;
1299         }
1300
1301         nobj->key = key;
1302         nobj->keylen = keylen;
1303
1304         if (!ucl_parser_process_object_element (parser, nobj)) {
1305                 return false;
1306         }
1307
1308         if (ucl_escape) {
1309                 nobj->flags |= UCL_OBJECT_NEED_KEY_ESCAPE;
1310         }
1311
1312
1313         return true;
1314 }
1315
1316 /**
1317  * Parse a cl string
1318  * @param parser
1319  * @param chunk
1320  * @param var_expand
1321  * @param need_unescape
1322  * @return true if a key has been parsed
1323  */
1324 static bool
1325 ucl_parse_string_value (struct ucl_parser *parser,
1326                 struct ucl_chunk *chunk, bool *var_expand, bool *need_unescape)
1327 {
1328         const unsigned char *p;
1329         enum {
1330                 UCL_BRACE_ROUND = 0,
1331                 UCL_BRACE_SQUARE,
1332                 UCL_BRACE_FIGURE
1333         };
1334         int braces[3][2] = {{0, 0}, {0, 0}, {0, 0}};
1335
1336         p = chunk->pos;
1337
1338         while (p < chunk->end) {
1339
1340                 /* Skip pairs of figure braces */
1341                 if (*p == '{') {
1342                         braces[UCL_BRACE_FIGURE][0] ++;
1343                 }
1344                 else if (*p == '}') {
1345                         braces[UCL_BRACE_FIGURE][1] ++;
1346                         if (braces[UCL_BRACE_FIGURE][1] <= braces[UCL_BRACE_FIGURE][0]) {
1347                                 /* This is not a termination symbol, continue */
1348                                 ucl_chunk_skipc (chunk, p);
1349                                 continue;
1350                         }
1351                 }
1352                 /* Skip pairs of square braces */
1353                 else if (*p == '[') {
1354                         braces[UCL_BRACE_SQUARE][0] ++;
1355                 }
1356                 else if (*p == ']') {
1357                         braces[UCL_BRACE_SQUARE][1] ++;
1358                         if (braces[UCL_BRACE_SQUARE][1] <= braces[UCL_BRACE_SQUARE][0]) {
1359                                 /* This is not a termination symbol, continue */
1360                                 ucl_chunk_skipc (chunk, p);
1361                                 continue;
1362                         }
1363                 }
1364                 else if (*p == '$') {
1365                         *var_expand = true;
1366                 }
1367                 else if (*p == '\\') {
1368                         *need_unescape = true;
1369                         ucl_chunk_skipc (chunk, p);
1370                         if (p < chunk->end) {
1371                                 ucl_chunk_skipc (chunk, p);
1372                         }
1373                         continue;
1374                 }
1375
1376                 if (ucl_lex_is_atom_end (*p) || (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1]))) {
1377                         break;
1378                 }
1379                 ucl_chunk_skipc (chunk, p);
1380         }
1381
1382         return true;
1383 }
1384
1385 /**
1386  * Parse multiline string ending with \n{term}\n
1387  * @param parser
1388  * @param chunk
1389  * @param term
1390  * @param term_len
1391  * @param beg
1392  * @param var_expand
1393  * @return size of multiline string or 0 in case of error
1394  */
1395 static int
1396 ucl_parse_multiline_string (struct ucl_parser *parser,
1397                 struct ucl_chunk *chunk, const unsigned char *term,
1398                 int term_len, unsigned char const **beg,
1399                 bool *var_expand)
1400 {
1401         const unsigned char *p, *c, *tend;
1402         bool newline = false;
1403         int len = 0;
1404
1405         p = chunk->pos;
1406
1407         c = p;
1408
1409         while (p < chunk->end) {
1410                 if (newline) {
1411                         if (chunk->end - p < term_len) {
1412                                 return 0;
1413                         }
1414                         else if (memcmp (p, term, term_len) == 0) {
1415                                 tend = p + term_len;
1416                                 if (*tend != '\n' && *tend != ';' && *tend != ',') {
1417                                         /* Incomplete terminator */
1418                                         ucl_chunk_skipc (chunk, p);
1419                                         continue;
1420                                 }
1421                                 len = p - c;
1422                                 chunk->remain -= term_len;
1423                                 chunk->pos = p + term_len;
1424                                 chunk->column = term_len;
1425                                 *beg = c;
1426                                 break;
1427                         }
1428                 }
1429                 if (*p == '\n') {
1430                         newline = true;
1431                 }
1432                 else {
1433                         if (*p == '$') {
1434                                 *var_expand = true;
1435                         }
1436                         newline = false;
1437                 }
1438                 ucl_chunk_skipc (chunk, p);
1439         }
1440
1441         return len;
1442 }
1443
1444 static inline ucl_object_t*
1445 ucl_parser_get_container (struct ucl_parser *parser)
1446 {
1447         ucl_object_t *t, *obj = NULL;
1448
1449         if (parser == NULL || parser->stack == NULL || parser->stack->obj == NULL) {
1450                 return NULL;
1451         }
1452
1453         if (parser->stack->obj->type == UCL_ARRAY) {
1454                 /* Object must be allocated */
1455                 obj = ucl_object_new_full (UCL_NULL, parser->chunks->priority);
1456                 t = parser->stack->obj;
1457
1458                 if (!ucl_array_append (t, obj)) {
1459                         ucl_object_unref (obj);
1460                         return NULL;
1461                 }
1462
1463                 parser->cur_obj = obj;
1464         }
1465         else {
1466                 /* Object has been already allocated */
1467                 obj = parser->cur_obj;
1468         }
1469
1470         return obj;
1471 }
1472
1473 /**
1474  * Handle value data
1475  * @param parser
1476  * @param chunk
1477  * @return
1478  */
1479 static bool
1480 ucl_parse_value (struct ucl_parser *parser, struct ucl_chunk *chunk)
1481 {
1482         const unsigned char *p, *c;
1483         ucl_object_t *obj = NULL;
1484         unsigned int stripped_spaces;
1485         int str_len;
1486         bool need_unescape = false, ucl_escape = false, var_expand = false;
1487
1488         p = chunk->pos;
1489
1490         /* Skip any spaces and comments */
1491         if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) ||
1492                         (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1]))) {
1493                 while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
1494                         ucl_chunk_skipc (chunk, p);
1495                 }
1496                 if (!ucl_skip_comments (parser)) {
1497                         return false;
1498                 }
1499                 p = chunk->pos;
1500         }
1501
1502         while (p < chunk->end) {
1503                 c = p;
1504                 switch (*p) {
1505                 case '"':
1506                         ucl_chunk_skipc (chunk, p);
1507
1508                         if (!ucl_lex_json_string (parser, chunk, &need_unescape, &ucl_escape,
1509                                         &var_expand)) {
1510                                 return false;
1511                         }
1512
1513                         obj = ucl_parser_get_container (parser);
1514                         str_len = chunk->pos - c - 2;
1515                         obj->type = UCL_STRING;
1516                         if ((str_len = ucl_copy_or_store_ptr (parser, c + 1,
1517                                         &obj->trash_stack[UCL_TRASH_VALUE],
1518                                         &obj->value.sv, str_len, need_unescape, false,
1519                                         var_expand)) == -1) {
1520                                 return false;
1521                         }
1522                         obj->len = str_len;
1523
1524                         parser->state = UCL_STATE_AFTER_VALUE;
1525                         p = chunk->pos;
1526
1527                         return true;
1528                         break;
1529                 case '{':
1530                         obj = ucl_parser_get_container (parser);
1531                         /* We have a new object */
1532                         obj = ucl_parser_add_container (obj, parser, false, parser->stack->level);
1533                         if (obj == NULL) {
1534                                 return false;
1535                         }
1536
1537                         ucl_chunk_skipc (chunk, p);
1538
1539                         return true;
1540                         break;
1541                 case '[':
1542                         obj = ucl_parser_get_container (parser);
1543                         /* We have a new array */
1544                         obj = ucl_parser_add_container (obj, parser, true, parser->stack->level);
1545                         if (obj == NULL) {
1546                                 return false;
1547                         }
1548
1549                         ucl_chunk_skipc (chunk, p);
1550
1551                         return true;
1552                         break;
1553                 case ']':
1554                         /* We have the array ending */
1555                         if (parser->stack && parser->stack->obj->type == UCL_ARRAY) {
1556                                 parser->state = UCL_STATE_AFTER_VALUE;
1557                                 return true;
1558                         }
1559                         else {
1560                                 goto parse_string;
1561                         }
1562                         break;
1563                 case '<':
1564                         obj = ucl_parser_get_container (parser);
1565                         /* We have something like multiline value, which must be <<[A-Z]+\n */
1566                         if (chunk->end - p > 3) {
1567                                 if (memcmp (p, "<<", 2) == 0) {
1568                                         p += 2;
1569                                         /* We allow only uppercase characters in multiline definitions */
1570                                         while (p < chunk->end && *p >= 'A' && *p <= 'Z') {
1571                                                 p ++;
1572                                         }
1573                                         if (*p =='\n') {
1574                                                 /* Set chunk positions and start multiline parsing */
1575                                                 c += 2;
1576                                                 chunk->remain -= p - c;
1577                                                 chunk->pos = p + 1;
1578                                                 chunk->column = 0;
1579                                                 chunk->line ++;
1580                                                 if ((str_len = ucl_parse_multiline_string (parser, chunk, c,
1581                                                                 p - c, &c, &var_expand)) == 0) {
1582                                                         ucl_set_err (parser, UCL_ESYNTAX,
1583                                                                         "unterminated multiline value", &parser->err);
1584                                                         return false;
1585                                                 }
1586
1587                                                 obj->type = UCL_STRING;
1588                                                 obj->flags |= UCL_OBJECT_MULTILINE;
1589                                                 if ((str_len = ucl_copy_or_store_ptr (parser, c,
1590                                                                 &obj->trash_stack[UCL_TRASH_VALUE],
1591                                                                 &obj->value.sv, str_len - 1, false,
1592                                                                 false, var_expand)) == -1) {
1593                                                         return false;
1594                                                 }
1595                                                 obj->len = str_len;
1596
1597                                                 parser->state = UCL_STATE_AFTER_VALUE;
1598
1599                                                 return true;
1600                                         }
1601                                 }
1602                         }
1603                         /* Fallback to ordinary strings */
1604                 default:
1605 parse_string:
1606                         if (obj == NULL) {
1607                                 obj = ucl_parser_get_container (parser);
1608                         }
1609
1610                         /* Parse atom */
1611                         if (ucl_test_character (*p, UCL_CHARACTER_VALUE_DIGIT_START)) {
1612                                 if (!ucl_lex_number (parser, chunk, obj)) {
1613                                         if (parser->state == UCL_STATE_ERROR) {
1614                                                 return false;
1615                                         }
1616                                 }
1617                                 else {
1618                                         parser->state = UCL_STATE_AFTER_VALUE;
1619                                         return true;
1620                                 }
1621                                 /* Fallback to normal string */
1622                         }
1623
1624                         if (!ucl_parse_string_value (parser, chunk, &var_expand,
1625                                         &need_unescape)) {
1626                                 return false;
1627                         }
1628                         /* Cut trailing spaces */
1629                         stripped_spaces = 0;
1630                         while (ucl_test_character (*(chunk->pos - 1 - stripped_spaces),
1631                                         UCL_CHARACTER_WHITESPACE)) {
1632                                 stripped_spaces ++;
1633                         }
1634                         str_len = chunk->pos - c - stripped_spaces;
1635                         if (str_len <= 0) {
1636                                 ucl_set_err (parser, UCL_ESYNTAX, "string value must not be empty",
1637                                                 &parser->err);
1638                                 return false;
1639                         }
1640                         else if (str_len == 4 && memcmp (c, "null", 4) == 0) {
1641                                 obj->len = 0;
1642                                 obj->type = UCL_NULL;
1643                         }
1644                         else if (!ucl_maybe_parse_boolean (obj, c, str_len)) {
1645                                 obj->type = UCL_STRING;
1646                                 if ((str_len = ucl_copy_or_store_ptr (parser, c,
1647                                                 &obj->trash_stack[UCL_TRASH_VALUE],
1648                                                 &obj->value.sv, str_len, need_unescape,
1649                                                 false, var_expand)) == -1) {
1650                                         return false;
1651                                 }
1652                                 obj->len = str_len;
1653                         }
1654                         parser->state = UCL_STATE_AFTER_VALUE;
1655                         p = chunk->pos;
1656
1657                         return true;
1658                         break;
1659                 }
1660         }
1661
1662         return true;
1663 }
1664
1665 /**
1666  * Handle after value data
1667  * @param parser
1668  * @param chunk
1669  * @return
1670  */
1671 static bool
1672 ucl_parse_after_value (struct ucl_parser *parser, struct ucl_chunk *chunk)
1673 {
1674         const unsigned char *p;
1675         bool got_sep = false;
1676         struct ucl_stack *st;
1677
1678         p = chunk->pos;
1679
1680         while (p < chunk->end) {
1681                 if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE)) {
1682                         /* Skip whitespaces */
1683                         ucl_chunk_skipc (chunk, p);
1684                 }
1685                 else if (chunk->remain >= 2 && ucl_lex_is_comment (p[0], p[1])) {
1686                         /* Skip comment */
1687                         if (!ucl_skip_comments (parser)) {
1688                                 return false;
1689                         }
1690                         /* Treat comment as a separator */
1691                         got_sep = true;
1692                         p = chunk->pos;
1693                 }
1694                 else if (ucl_test_character (*p, UCL_CHARACTER_VALUE_END)) {
1695                         if (*p == '}' || *p == ']') {
1696                                 if (parser->stack == NULL) {
1697                                         ucl_set_err (parser, UCL_ESYNTAX,
1698                                                         "end of array or object detected without corresponding start",
1699                                                         &parser->err);
1700                                         return false;
1701                                 }
1702                                 if ((*p == '}' && parser->stack->obj->type == UCL_OBJECT) ||
1703                                                 (*p == ']' && parser->stack->obj->type == UCL_ARRAY)) {
1704
1705                                         /* Pop all nested objects from a stack */
1706                                         st = parser->stack;
1707                                         parser->stack = st->next;
1708                                         UCL_FREE (sizeof (struct ucl_stack), st);
1709
1710                                         while (parser->stack != NULL) {
1711                                                 st = parser->stack;
1712                                                 if (st->next == NULL || st->next->level == st->level) {
1713                                                         break;
1714                                                 }
1715                                                 parser->stack = st->next;
1716                                                 UCL_FREE (sizeof (struct ucl_stack), st);
1717                                         }
1718                                 }
1719                                 else {
1720                                         ucl_set_err (parser, UCL_ESYNTAX,
1721                                                         "unexpected terminating symbol detected",
1722                                                         &parser->err);
1723                                         return false;
1724                                 }
1725
1726                                 if (parser->stack == NULL) {
1727                                         /* Ignore everything after a top object */
1728                                         return true;
1729                                 }
1730                                 else {
1731                                         ucl_chunk_skipc (chunk, p);
1732                                 }
1733                                 got_sep = true;
1734                         }
1735                         else {
1736                                 /* Got a separator */
1737                                 got_sep = true;
1738                                 ucl_chunk_skipc (chunk, p);
1739                         }
1740                 }
1741                 else {
1742                         /* Anything else */
1743                         if (!got_sep) {
1744                                 ucl_set_err (parser, UCL_ESYNTAX, "delimiter is missing",
1745                                                 &parser->err);
1746                                 return false;
1747                         }
1748                         return true;
1749                 }
1750         }
1751
1752         return true;
1753 }
1754
1755 /**
1756  * Handle macro data
1757  * @param parser
1758  * @param chunk
1759  * @param marco
1760  * @param macro_start
1761  * @param macro_len
1762  * @return
1763  */
1764 static bool
1765 ucl_parse_macro_value (struct ucl_parser *parser,
1766                 struct ucl_chunk *chunk, struct ucl_macro *macro,
1767                 unsigned char const **macro_start, size_t *macro_len)
1768 {
1769         const unsigned char *p, *c;
1770         bool need_unescape = false, ucl_escape = false, var_expand = false;
1771
1772         p = chunk->pos;
1773
1774         switch (*p) {
1775         case '"':
1776                 /* We have macro value encoded in quotes */
1777                 c = p;
1778                 ucl_chunk_skipc (chunk, p);
1779                 if (!ucl_lex_json_string (parser, chunk, &need_unescape, &ucl_escape, &var_expand)) {
1780                         return false;
1781                 }
1782
1783                 *macro_start = c + 1;
1784                 *macro_len = chunk->pos - c - 2;
1785                 p = chunk->pos;
1786                 break;
1787         case '{':
1788                 /* We got a multiline macro body */
1789                 ucl_chunk_skipc (chunk, p);
1790                 /* Skip spaces at the beginning */
1791                 while (p < chunk->end) {
1792                         if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
1793                                 ucl_chunk_skipc (chunk, p);
1794                         }
1795                         else {
1796                                 break;
1797                         }
1798                 }
1799                 c = p;
1800                 while (p < chunk->end) {
1801                         if (*p == '}') {
1802                                 break;
1803                         }
1804                         ucl_chunk_skipc (chunk, p);
1805                 }
1806                 *macro_start = c;
1807                 *macro_len = p - c;
1808                 ucl_chunk_skipc (chunk, p);
1809                 break;
1810         default:
1811                 /* Macro is not enclosed in quotes or braces */
1812                 c = p;
1813                 while (p < chunk->end) {
1814                         if (ucl_lex_is_atom_end (*p)) {
1815                                 break;
1816                         }
1817                         ucl_chunk_skipc (chunk, p);
1818                 }
1819                 *macro_start = c;
1820                 *macro_len = p - c;
1821                 break;
1822         }
1823
1824         /* We are at the end of a macro */
1825         /* Skip ';' and space characters and return to previous state */
1826         while (p < chunk->end) {
1827                 if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) && *p != ';') {
1828                         break;
1829                 }
1830                 ucl_chunk_skipc (chunk, p);
1831         }
1832         return true;
1833 }
1834
1835 /**
1836  * Parse macro arguments as UCL object
1837  * @param parser parser structure
1838  * @param chunk the current data chunk
1839  * @return
1840  */
1841 static ucl_object_t *
1842 ucl_parse_macro_arguments (struct ucl_parser *parser,
1843                 struct ucl_chunk *chunk)
1844 {
1845         ucl_object_t *res = NULL;
1846         struct ucl_parser *params_parser;
1847         int obraces = 1, ebraces = 0, state = 0;
1848         const unsigned char *p, *c;
1849         size_t args_len = 0;
1850         struct ucl_parser_saved_state saved;
1851
1852         saved.column = chunk->column;
1853         saved.line = chunk->line;
1854         saved.pos = chunk->pos;
1855         saved.remain = chunk->remain;
1856         p = chunk->pos;
1857
1858         if (*p != '(' || chunk->remain < 2) {
1859                 return NULL;
1860         }
1861
1862         /* Set begin and start */
1863         ucl_chunk_skipc (chunk, p);
1864         c = p;
1865
1866         while ((p) < (chunk)->end) {
1867                 switch (state) {
1868                 case 0:
1869                         /* Parse symbols and check for '(', ')' and '"' */
1870                         if (*p == '(') {
1871                                 obraces ++;
1872                         }
1873                         else if (*p == ')') {
1874                                 ebraces ++;
1875                         }
1876                         else if (*p == '"') {
1877                                 state = 1;
1878                         }
1879                         /* Check pairing */
1880                         if (obraces == ebraces) {
1881                                 state = 99;
1882                         }
1883                         else {
1884                                 args_len ++;
1885                         }
1886                         /* Check overflow */
1887                         if (chunk->remain == 0) {
1888                                 goto restore_chunk;
1889                         }
1890                         ucl_chunk_skipc (chunk, p);
1891                         break;
1892                 case 1:
1893                         /* We have quote character, so skip all but quotes */
1894                         if (*p == '"' && *(p - 1) != '\\') {
1895                                 state = 0;
1896                         }
1897                         if (chunk->remain == 0) {
1898                                 goto restore_chunk;
1899                         }
1900                         args_len ++;
1901                         ucl_chunk_skipc (chunk, p);
1902                         break;
1903                 case 99:
1904                         /*
1905                          * We have read the full body of arguments, so we need to parse and set
1906                          * object from that
1907                          */
1908                         params_parser = ucl_parser_new (parser->flags);
1909                         if (!ucl_parser_add_chunk (params_parser, c, args_len)) {
1910                                 ucl_set_err (parser, UCL_ESYNTAX, "macro arguments parsing error",
1911                                                 &parser->err);
1912                         }
1913                         else {
1914                                 res = ucl_parser_get_object (params_parser);
1915                         }
1916                         ucl_parser_free (params_parser);
1917
1918                         return res;
1919
1920                         break;
1921                 }
1922         }
1923
1924         return res;
1925
1926 restore_chunk:
1927         chunk->column = saved.column;
1928         chunk->line = saved.line;
1929         chunk->pos = saved.pos;
1930         chunk->remain = saved.remain;
1931
1932         return NULL;
1933 }
1934
1935 #define SKIP_SPACES_COMMENTS(parser, chunk, p) do {                                                             \
1936         while ((p) < (chunk)->end) {                                                                                            \
1937                 if (!ucl_test_character (*(p), UCL_CHARACTER_WHITESPACE_UNSAFE)) {              \
1938                         if ((chunk)->remain >= 2 && ucl_lex_is_comment ((p)[0], (p)[1])) {      \
1939                                 if (!ucl_skip_comments (parser)) {                                                              \
1940                                         return false;                                                                                           \
1941                                 }                                                                                                                               \
1942                                 p = (chunk)->pos;                                                                                               \
1943                         }                                                                                                                                       \
1944                         break;                                                                                                                          \
1945                 }                                                                                                                                               \
1946                 ucl_chunk_skipc (chunk, p);                                                                                             \
1947         }                                                                                                                                                       \
1948 } while(0)
1949
1950 /**
1951  * Handle the main states of rcl parser
1952  * @param parser parser structure
1953  * @return true if chunk has been parsed and false in case of error
1954  */
1955 static bool
1956 ucl_state_machine (struct ucl_parser *parser)
1957 {
1958         ucl_object_t *obj, *macro_args;
1959         struct ucl_chunk *chunk = parser->chunks;
1960         const unsigned char *p, *c = NULL, *macro_start = NULL;
1961         unsigned char *macro_escaped;
1962         size_t macro_len = 0;
1963         struct ucl_macro *macro = NULL;
1964         bool next_key = false, end_of_object = false, ret;
1965
1966         if (parser->top_obj == NULL) {
1967                 parser->state = UCL_STATE_INIT;
1968         }
1969
1970         p = chunk->pos;
1971         while (chunk->pos < chunk->end) {
1972                 switch (parser->state) {
1973                 case UCL_STATE_INIT:
1974                         /*
1975                          * At the init state we can either go to the parse array or object
1976                          * if we got [ or { correspondingly or can just treat new data as
1977                          * a key of newly created object
1978                          */
1979                         if (!ucl_skip_comments (parser)) {
1980                                 parser->prev_state = parser->state;
1981                                 parser->state = UCL_STATE_ERROR;
1982                                 return false;
1983                         }
1984                         else {
1985                                 /* Skip any spaces */
1986                                 while (p < chunk->end && ucl_test_character (*p,
1987                                                 UCL_CHARACTER_WHITESPACE_UNSAFE)) {
1988                                         ucl_chunk_skipc (chunk, p);
1989                                 }
1990
1991                                 p = chunk->pos;
1992
1993                                 if (*p == '[') {
1994                                         parser->state = UCL_STATE_VALUE;
1995                                         ucl_chunk_skipc (chunk, p);
1996                                 }
1997                                 else {
1998                                         parser->state = UCL_STATE_KEY;
1999                                         if (*p == '{') {
2000                                                 ucl_chunk_skipc (chunk, p);
2001                                         }
2002                                 }
2003
2004                                 if (parser->top_obj == NULL) {
2005                                         if (parser->state == UCL_STATE_VALUE) {
2006                                                 obj = ucl_parser_add_container (NULL, parser, true, 0);
2007                                         }
2008                                         else {
2009                                                 obj = ucl_parser_add_container (NULL, parser, false, 0);
2010                                         }
2011
2012                                         if (obj == NULL) {
2013                                                 return false;
2014                                         }
2015
2016                                         parser->top_obj = obj;
2017                                         parser->cur_obj = obj;
2018                                 }
2019
2020                         }
2021                         break;
2022                 case UCL_STATE_KEY:
2023                         /* Skip any spaces */
2024                         while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
2025                                 ucl_chunk_skipc (chunk, p);
2026                         }
2027                         if (*p == '}') {
2028                                 /* We have the end of an object */
2029                                 parser->state = UCL_STATE_AFTER_VALUE;
2030                                 continue;
2031                         }
2032                         if (parser->stack == NULL) {
2033                                 /* No objects are on stack, but we want to parse a key */
2034                                 ucl_set_err (parser, UCL_ESYNTAX, "top object is finished but the parser "
2035                                                 "expects a key", &parser->err);
2036                                 parser->prev_state = parser->state;
2037                                 parser->state = UCL_STATE_ERROR;
2038                                 return false;
2039                         }
2040                         if (!ucl_parse_key (parser, chunk, &next_key, &end_of_object)) {
2041                                 parser->prev_state = parser->state;
2042                                 parser->state = UCL_STATE_ERROR;
2043                                 return false;
2044                         }
2045                         if (end_of_object) {
2046                                 p = chunk->pos;
2047                                 parser->state = UCL_STATE_AFTER_VALUE;
2048                                 continue;
2049                         }
2050                         else if (parser->state != UCL_STATE_MACRO_NAME) {
2051                                 if (next_key && parser->stack->obj->type == UCL_OBJECT) {
2052                                         /* Parse more keys and nest objects accordingly */
2053                                         obj = ucl_parser_add_container (parser->cur_obj, parser, false,
2054                                                         parser->stack->level + 1);
2055                                         if (obj == NULL) {
2056                                                 return false;
2057                                         }
2058                                 }
2059                                 else {
2060                                         parser->state = UCL_STATE_VALUE;
2061                                 }
2062                         }
2063                         else {
2064                                 c = chunk->pos;
2065                         }
2066                         p = chunk->pos;
2067                         break;
2068                 case UCL_STATE_VALUE:
2069                         /* We need to check what we do have */
2070                         if (!ucl_parse_value (parser, chunk)) {
2071                                 parser->prev_state = parser->state;
2072                                 parser->state = UCL_STATE_ERROR;
2073                                 return false;
2074                         }
2075                         /* State is set in ucl_parse_value call */
2076                         p = chunk->pos;
2077                         break;
2078                 case UCL_STATE_AFTER_VALUE:
2079                         if (!ucl_parse_after_value (parser, chunk)) {
2080                                 parser->prev_state = parser->state;
2081                                 parser->state = UCL_STATE_ERROR;
2082                                 return false;
2083                         }
2084
2085                         if (parser->stack != NULL) {
2086                                 if (parser->stack->obj->type == UCL_OBJECT) {
2087                                         parser->state = UCL_STATE_KEY;
2088                                 }
2089                                 else {
2090                                         /* Array */
2091                                         parser->state = UCL_STATE_VALUE;
2092                                 }
2093                         }
2094                         else {
2095                                 /* Skip everything at the end */
2096                                 return true;
2097                         }
2098                         p = chunk->pos;
2099                         break;
2100                 case UCL_STATE_MACRO_NAME:
2101                         if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) &&
2102                                         *p != '(') {
2103                                 ucl_chunk_skipc (chunk, p);
2104                         }
2105                         else {
2106                                 if (p - c > 0) {
2107                                         /* We got macro name */
2108                                         macro_len = (size_t) (p - c);
2109                                         HASH_FIND (hh, parser->macroes, c, macro_len, macro);
2110                                         if (macro == NULL) {
2111                                                 ucl_create_err (&parser->err,
2112                                                                 "error on line %d at column %d: "
2113                                                                                 "unknown macro: '%.*s', character: '%c'",
2114                                                                 chunk->line,
2115                                                                 chunk->column,
2116                                                                 (int) (p - c),
2117                                                                 c,
2118                                                                 *chunk->pos);
2119                                                 parser->state = UCL_STATE_ERROR;
2120                                                 return false;
2121                                         }
2122                                         /* Now we need to skip all spaces */
2123                                         SKIP_SPACES_COMMENTS(parser, chunk, p);
2124                                         parser->state = UCL_STATE_MACRO;
2125                                 }
2126                                 else {
2127                                         /* We have invalid macro name */
2128                                         ucl_create_err (&parser->err,
2129                                                         "error on line %d at column %d: invalid macro name",
2130                                                         chunk->line,
2131                                                         chunk->column);
2132                                         parser->state = UCL_STATE_ERROR;
2133                                         return false;
2134                                 }
2135                         }
2136                         break;
2137                 case UCL_STATE_MACRO:
2138                         if (*chunk->pos == '(') {
2139                                 macro_args = ucl_parse_macro_arguments (parser, chunk);
2140                                 p = chunk->pos;
2141                                 if (macro_args) {
2142                                         SKIP_SPACES_COMMENTS(parser, chunk, p);
2143                                 }
2144                         }
2145                         else {
2146                                 macro_args = NULL;
2147                         }
2148                         if (!ucl_parse_macro_value (parser, chunk, macro,
2149                                         &macro_start, &macro_len)) {
2150                                 parser->prev_state = parser->state;
2151                                 parser->state = UCL_STATE_ERROR;
2152                                 return false;
2153                         }
2154                         macro_len = ucl_expand_variable (parser, &macro_escaped,
2155                                         macro_start, macro_len);
2156                         parser->state = parser->prev_state;
2157                         if (macro_escaped == NULL) {
2158                                 if (macro->is_context) {
2159                                         ret = macro->h.context_handler (macro_start, macro_len,
2160                                                         macro_args,
2161                                                         parser->top_obj,
2162                                                         macro->ud);
2163                                 }
2164                                 else {
2165                                         ret = macro->h.handler (macro_start, macro_len, macro_args,
2166                                                         macro->ud);
2167                                 }
2168                         }
2169                         else {
2170                                 if (macro->is_context) {
2171                                         ret = macro->h.context_handler (macro_escaped, macro_len,
2172                                                         macro_args,
2173                                                         parser->top_obj,
2174                                                         macro->ud);
2175                                 }
2176                                 else {
2177                                         ret = macro->h.handler (macro_escaped, macro_len, macro_args,
2178                                                 macro->ud);
2179                                 }
2180
2181                                 UCL_FREE (macro_len + 1, macro_escaped);
2182                         }
2183
2184                         /*
2185                          * Chunk can be modified within macro handler
2186                          */
2187                         chunk = parser->chunks;
2188                         p = chunk->pos;
2189                         if (macro_args) {
2190                                 ucl_object_unref (macro_args);
2191                         }
2192                         if (!ret) {
2193                                 return false;
2194                         }
2195                         break;
2196                 default:
2197                         /* TODO: add all states */
2198                         ucl_set_err (parser, UCL_EINTERNAL,
2199                                         "internal error: parser is in an unknown state", &parser->err);
2200                         parser->state = UCL_STATE_ERROR;
2201                         return false;
2202                 }
2203         }
2204
2205         return true;
2206 }
2207
2208 struct ucl_parser*
2209 ucl_parser_new (int flags)
2210 {
2211         struct ucl_parser *new;
2212
2213         new = UCL_ALLOC (sizeof (struct ucl_parser));
2214         if (new == NULL) {
2215                 return NULL;
2216         }
2217
2218         memset (new, 0, sizeof (struct ucl_parser));
2219
2220         ucl_parser_register_macro (new, "include", ucl_include_handler, new);
2221         ucl_parser_register_macro (new, "try_include", ucl_try_include_handler, new);
2222         ucl_parser_register_macro (new, "includes", ucl_includes_handler, new);
2223         ucl_parser_register_macro (new, "priority", ucl_priority_handler, new);
2224         ucl_parser_register_macro (new, "load", ucl_load_handler, new);
2225         ucl_parser_register_context_macro (new, "inherit", ucl_inherit_handler, new);
2226
2227         new->flags = flags;
2228         new->includepaths = NULL;
2229
2230         /* Initial assumption about filevars */
2231         ucl_parser_set_filevars (new, NULL, false);
2232
2233         return new;
2234 }
2235
2236 bool
2237 ucl_parser_set_default_priority (struct ucl_parser *parser, unsigned prio)
2238 {
2239         if (parser == NULL) {
2240                 return false;
2241         }
2242
2243         parser->default_priority = prio;
2244
2245         return true;
2246 }
2247
2248 void
2249 ucl_parser_register_macro (struct ucl_parser *parser, const char *macro,
2250                 ucl_macro_handler handler, void* ud)
2251 {
2252         struct ucl_macro *new;
2253
2254         if (macro == NULL || handler == NULL) {
2255                 return;
2256         }
2257
2258         new = UCL_ALLOC (sizeof (struct ucl_macro));
2259         if (new == NULL) {
2260                 return;
2261         }
2262
2263         memset (new, 0, sizeof (struct ucl_macro));
2264         new->h.handler = handler;
2265         new->name = strdup (macro);
2266         new->ud = ud;
2267         HASH_ADD_KEYPTR (hh, parser->macroes, new->name, strlen (new->name), new);
2268 }
2269
2270 void
2271 ucl_parser_register_context_macro (struct ucl_parser *parser, const char *macro,
2272                 ucl_context_macro_handler handler, void* ud)
2273 {
2274         struct ucl_macro *new;
2275
2276         if (macro == NULL || handler == NULL) {
2277                 return;
2278         }
2279
2280         new = UCL_ALLOC (sizeof (struct ucl_macro));
2281         if (new == NULL) {
2282                 return;
2283         }
2284
2285         memset (new, 0, sizeof (struct ucl_macro));
2286         new->h.context_handler = handler;
2287         new->name = strdup (macro);
2288         new->ud = ud;
2289         new->is_context = true;
2290         HASH_ADD_KEYPTR (hh, parser->macroes, new->name, strlen (new->name), new);
2291 }
2292
2293 void
2294 ucl_parser_register_variable (struct ucl_parser *parser, const char *var,
2295                 const char *value)
2296 {
2297         struct ucl_variable *new = NULL, *cur;
2298
2299         if (var == NULL) {
2300                 return;
2301         }
2302
2303         /* Find whether a variable already exists */
2304         LL_FOREACH (parser->variables, cur) {
2305                 if (strcmp (cur->var, var) == 0) {
2306                         new = cur;
2307                         break;
2308                 }
2309         }
2310
2311         if (value == NULL) {
2312
2313                 if (new != NULL) {
2314                         /* Remove variable */
2315                         DL_DELETE (parser->variables, new);
2316                         free (new->var);
2317                         free (new->value);
2318                         UCL_FREE (sizeof (struct ucl_variable), new);
2319                 }
2320                 else {
2321                         /* Do nothing */
2322                         return;
2323                 }
2324         }
2325         else {
2326                 if (new == NULL) {
2327                         new = UCL_ALLOC (sizeof (struct ucl_variable));
2328                         if (new == NULL) {
2329                                 return;
2330                         }
2331                         memset (new, 0, sizeof (struct ucl_variable));
2332                         new->var = strdup (var);
2333                         new->var_len = strlen (var);
2334                         new->value = strdup (value);
2335                         new->value_len = strlen (value);
2336
2337                         DL_APPEND (parser->variables, new);
2338                 }
2339                 else {
2340                         free (new->value);
2341                         new->value = strdup (value);
2342                         new->value_len = strlen (value);
2343                 }
2344         }
2345 }
2346
2347 void
2348 ucl_parser_set_variables_handler (struct ucl_parser *parser,
2349                 ucl_variable_handler handler, void *ud)
2350 {
2351         parser->var_handler = handler;
2352         parser->var_data = ud;
2353 }
2354
2355 bool
2356 ucl_parser_add_chunk_full (struct ucl_parser *parser, const unsigned char *data,
2357                 size_t len, unsigned priority, enum ucl_duplicate_strategy strat,
2358                 enum ucl_parse_type parse_type)
2359 {
2360         struct ucl_chunk *chunk;
2361
2362         if (parser == NULL) {
2363                 return false;
2364         }
2365
2366         if (data == NULL) {
2367                 ucl_create_err (&parser->err, "invalid chunk added");
2368                 return false;
2369         }
2370         if (len == 0) {
2371                 parser->top_obj = ucl_object_new_full (UCL_OBJECT, priority);
2372                 return true;
2373         }
2374         if (parser->state != UCL_STATE_ERROR) {
2375                 chunk = UCL_ALLOC (sizeof (struct ucl_chunk));
2376                 if (chunk == NULL) {
2377                         ucl_create_err (&parser->err, "cannot allocate chunk structure");
2378                         return false;
2379                 }
2380                 chunk->begin = data;
2381                 chunk->remain = len;
2382                 chunk->pos = chunk->begin;
2383                 chunk->end = chunk->begin + len;
2384                 chunk->line = 1;
2385                 chunk->column = 0;
2386                 chunk->priority = priority;
2387                 chunk->strategy = strat;
2388                 chunk->parse_type = parse_type;
2389                 LL_PREPEND (parser->chunks, chunk);
2390                 parser->recursion ++;
2391
2392                 if (parser->recursion > UCL_MAX_RECURSION) {
2393                         ucl_create_err (&parser->err, "maximum include nesting limit is reached: %d",
2394                                         parser->recursion);
2395                         return false;
2396                 }
2397
2398                 switch (parse_type) {
2399                 default:
2400                 case UCL_PARSE_UCL:
2401                         return ucl_state_machine (parser);
2402                 case UCL_PARSE_MSGPACK:
2403                         return ucl_parse_msgpack (parser);
2404                 }
2405         }
2406
2407         ucl_create_err (&parser->err, "a parser is in an invalid state");
2408
2409         return false;
2410 }
2411
2412 bool
2413 ucl_parser_add_chunk_priority (struct ucl_parser *parser,
2414                 const unsigned char *data, size_t len, unsigned priority)
2415 {
2416         /* We dereference parser, so this check is essential */
2417         if (parser == NULL) {
2418                 return false;
2419         }
2420
2421         return ucl_parser_add_chunk_full (parser, data, len,
2422                                 priority, UCL_DUPLICATE_APPEND, UCL_PARSE_UCL);
2423 }
2424
2425 bool
2426 ucl_parser_add_chunk (struct ucl_parser *parser, const unsigned char *data,
2427                 size_t len)
2428 {
2429         if (parser == NULL) {
2430                 return false;
2431         }
2432
2433         return ucl_parser_add_chunk_full (parser, data, len,
2434                         parser->default_priority, UCL_DUPLICATE_APPEND, UCL_PARSE_UCL);
2435 }
2436
2437 bool
2438 ucl_parser_add_string_priority (struct ucl_parser *parser, const char *data,
2439                 size_t len, unsigned priority)
2440 {
2441         if (data == NULL) {
2442                 ucl_create_err (&parser->err, "invalid string added");
2443                 return false;
2444         }
2445         if (len == 0) {
2446                 len = strlen (data);
2447         }
2448
2449         return ucl_parser_add_chunk_priority (parser,
2450                         (const unsigned char *)data, len, priority);
2451 }
2452
2453 bool
2454 ucl_parser_add_string (struct ucl_parser *parser, const char *data,
2455                 size_t len)
2456 {
2457         if (parser == NULL) {
2458                 return false;
2459         }
2460
2461         return ucl_parser_add_string_priority (parser,
2462                         (const unsigned char *)data, len, parser->default_priority);
2463 }
2464
2465 bool
2466 ucl_set_include_path (struct ucl_parser *parser, ucl_object_t *paths)
2467 {
2468         if (parser == NULL || paths == NULL) {
2469                 return false;
2470         }
2471
2472         if (parser->includepaths == NULL) {
2473                 parser->includepaths = ucl_object_copy (paths);
2474         }
2475         else {
2476                 ucl_object_unref (parser->includepaths);
2477                 parser->includepaths = ucl_object_copy (paths);
2478         }
2479
2480         if (parser->includepaths == NULL) {
2481                 return false;
2482         }
2483
2484         return true;
2485 }