contrib/libucl/src/ucl_util.c

   1 /* Copyright (c) 2013, Vsevolod Stakhov
   2  * Copyright (c) 2015 Allan Jude <allanjude@freebsd.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *       * Redistributions of source code must retain the above copyright
   8  *         notice, this list of conditions and the following disclaimer.
   9  *       * Redistributions in binary form must reproduce the above copyright
  10  *         notice, this list of conditions and the following disclaimer in the
  11  *         documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  15  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  16  * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
  17  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  18  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  19  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  20  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  22  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  23  */
  24
  25 #include "ucl.h"
  26 #include "ucl_internal.h"
  27 #include "ucl_chartable.h"
  28 #include "kvec.h"
  29 #include <limits.h>
  30 #include <stdarg.h>
  31 #include <stdio.h> /* for snprintf */
  32
  33 #ifndef _WIN32
  34 #include <glob.h>
  35 #include <sys/param.h>
  36 #else
  37 #ifndef NBBY
  38 #define NBBY 8
  39 #endif
  40 #endif
  41
  42 #ifdef HAVE_LIBGEN_H
  43 #include <libgen.h> /* For dirname */
  44 #endif
  45
  46 typedef kvec_t(ucl_object_t *) ucl_array_t;
  47
  48 #define UCL_ARRAY_GET(ar, obj) ucl_array_t *ar = \
  49         (ucl_array_t *)((obj) != NULL ? (obj)->value.av : NULL)
  50
  51 #ifdef HAVE_OPENSSL
  52 #include <openssl/err.h>
  53 #include <openssl/sha.h>
  54 #include <openssl/rsa.h>
  55 #include <openssl/ssl.h>
  56 #include <openssl/evp.h>
  57 #endif
  58
  59 #ifdef CURL_FOUND
  60 /* Seems to be broken */
  61 #define CURL_DISABLE_TYPECHECK 1
  62 #include <curl/curl.h>
  63 #endif
  64 #ifdef HAVE_FETCH_H
  65 #include <fetch.h>
  66 #endif
  67
  68 #ifdef _WIN32
  69 #include <windows.h>
  70
  71 #ifndef PROT_READ
  72 #define PROT_READ       1
  73 #endif
  74 #ifndef PROT_WRITE
  75 #define PROT_WRITE      2
  76 #endif
  77 #ifndef PROT_READWRITE
  78 #define PROT_READWRITE  3
  79 #endif
  80 #ifndef MAP_SHARED
  81 #define MAP_SHARED      1
  82 #endif
  83 #ifndef MAP_PRIVATE
  84 #define MAP_PRIVATE     2
  85 #endif
  86 #ifndef MAP_FAILED
  87 #define MAP_FAILED      ((void *) -1)
  88 #endif
  89
  90 static void *ucl_mmap(char *addr, size_t length, int prot, int access, int fd, off_t offset)
  91 {
  92         void *map = NULL;
  93         HANDLE handle = INVALID_HANDLE_VALUE;
  94
  95         switch (prot) {
  96         default:
  97         case PROT_READ:
  98                 {
  99                         handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READONLY, 0, length, 0);
 100                         if (!handle) break;
 101                         map = (void *) MapViewOfFile(handle, FILE_MAP_READ, 0, 0, length);
 102                         CloseHandle(handle);
 103                         break;
 104                 }
 105         case PROT_WRITE:
 106                 {
 107                         handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READWRITE, 0, length, 0);
 108                         if (!handle) break;
 109                         map = (void *) MapViewOfFile(handle, FILE_MAP_WRITE, 0, 0, length);
 110                         CloseHandle(handle);
 111                         break;
 112                 }
 113         case PROT_READWRITE:
 114                 {
 115                         handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READWRITE, 0, length, 0);
 116                         if (!handle) break;
 117                         map = (void *) MapViewOfFile(handle, FILE_MAP_ALL_ACCESS, 0, 0, length);
 118                         CloseHandle(handle);
 119                         break;
 120                 }
 121         }
 122         if (map == (void *) NULL) {
 123                 return (void *) MAP_FAILED;
 124         }
 125         return (void *) ((char *) map + offset);
 126 }
 127
 128 static int ucl_munmap(void *map,size_t length)
 129 {
 130         if (!UnmapViewOfFile(map)) {
 131                 return(-1);
 132         }
 133         return(0);
 134 }
 135
 136 static char* ucl_realpath(const char *path, char *resolved_path) {
 137     char *p;
 138     char tmp[MAX_PATH + 1];
 139     strncpy(tmp, path, sizeof(tmp)-1);
 140     p = tmp;
 141     while(*p) {
 142         if (*p == '/') *p = '\\';
 143         p++;
 144     }
 145     return _fullpath(resolved_path, tmp, MAX_PATH);
 146 }
 147 #else
 148 #define ucl_mmap mmap
 149 #define ucl_munmap munmap
 150 #define ucl_realpath realpath
 151 #endif
 152
 153 typedef void (*ucl_object_dtor) (ucl_object_t *obj);
 154 static void ucl_object_free_internal (ucl_object_t *obj, bool allow_rec,
 155                 ucl_object_dtor dtor);
 156 static void ucl_object_dtor_unref (ucl_object_t *obj);
 157
 158 static void
 159 ucl_object_dtor_free (ucl_object_t *obj)
 160 {
 161         if (obj->trash_stack[UCL_TRASH_KEY] != NULL) {
 162                 UCL_FREE (obj->hh.keylen, obj->trash_stack[UCL_TRASH_KEY]);
 163         }
 164         if (obj->trash_stack[UCL_TRASH_VALUE] != NULL) {
 165                 UCL_FREE (obj->len, obj->trash_stack[UCL_TRASH_VALUE]);
 166         }
 167         /* Do not free ephemeral objects */
 168         if ((obj->flags & UCL_OBJECT_EPHEMERAL) == 0) {
 169                 if (obj->type != UCL_USERDATA) {
 170                         UCL_FREE (sizeof (ucl_object_t), obj);
 171                 }
 172                 else {
 173                         struct ucl_object_userdata *ud = (struct ucl_object_userdata *)obj;
 174                         if (ud->dtor) {
 175                                 ud->dtor (obj->value.ud);
 176                         }
 177                         UCL_FREE (sizeof (*ud), obj);
 178                 }
 179         }
 180 }
 181
 182 /*
 183  * This is a helper function that performs exactly the same as
 184  * `ucl_object_unref` but it doesn't iterate over elements allowing
 185  * to use it for individual elements of arrays and multiple values
 186  */
 187 static void
 188 ucl_object_dtor_unref_single (ucl_object_t *obj)
 189 {
 190         if (obj != NULL) {
 191 #ifdef HAVE_ATOMIC_BUILTINS
 192                 unsigned int rc = __sync_sub_and_fetch (&obj->ref, 1);
 193                 if (rc == 0) {
 194 #else
 195                 if (--obj->ref == 0) {
 196 #endif
 197                         ucl_object_free_internal (obj, false, ucl_object_dtor_unref);
 198                 }
 199         }
 200 }
 201
 202 static void
 203 ucl_object_dtor_unref (ucl_object_t *obj)
 204 {
 205         if (obj->ref == 0) {
 206                 ucl_object_dtor_free (obj);
 207         }
 208         else {
 209                 /* This may cause dtor unref being called one more time */
 210                 ucl_object_dtor_unref_single (obj);
 211         }
 212 }
 213
 214 static void
 215 ucl_object_free_internal (ucl_object_t *obj, bool allow_rec, ucl_object_dtor dtor)
 216 {
 217         ucl_object_t *tmp, *sub;
 218
 219         while (obj != NULL) {
 220                 if (obj->type == UCL_ARRAY) {
 221                         UCL_ARRAY_GET (vec, obj);
 222                         unsigned int i;
 223
 224                         if (vec != NULL) {
 225                                 for (i = 0; i < vec->n; i ++) {
 226                                         sub = kv_A (*vec, i);
 227                                         if (sub != NULL) {
 228                                                 tmp = sub;
 229                                                 while (sub) {
 230                                                         tmp = sub->next;
 231                                                         dtor (sub);
 232                                                         sub = tmp;
 233                                                 }
 234                                         }
 235                                 }
 236                                 kv_destroy (*vec);
 237                                 UCL_FREE (sizeof (*vec), vec);
 238                         }
 239                         obj->value.av = NULL;
 240                 }
 241                 else if (obj->type == UCL_OBJECT) {
 242                         if (obj->value.ov != NULL) {
 243                                 ucl_hash_destroy (obj->value.ov, (ucl_hash_free_func)dtor);
 244                         }
 245                         obj->value.ov = NULL;
 246                 }
 247                 tmp = obj->next;
 248                 dtor (obj);
 249                 obj = tmp;
 250
 251                 if (!allow_rec) {
 252                         break;
 253                 }
 254         }
 255 }
 256
 257 void
 258 ucl_object_free (ucl_object_t *obj)
 259 {
 260         ucl_object_free_internal (obj, true, ucl_object_dtor_free);
 261 }
 262
 263 size_t
 264 ucl_unescape_json_string (char *str, size_t len)
 265 {
 266         char *t = str, *h = str;
 267         int i, uval;
 268
 269         if (len <= 1) {
 270                 return len;
 271         }
 272         /* t is target (tortoise), h is source (hare) */
 273
 274         while (len) {
 275                 if (*h == '\\') {
 276                         h ++;
 277
 278                         if (len == 1) {
 279                                 /*
 280                                  * If \ is last, then do not try to go further
 281                                  * Issue: #74
 282                                  */
 283                                 len --;
 284                                 *t++ = '\\';
 285                                 continue;
 286                         }
 287
 288                         switch (*h) {
 289                         case 'n':
 290                                 *t++ = '\n';
 291                                 break;
 292                         case 'r':
 293                                 *t++ = '\r';
 294                                 break;
 295                         case 'b':
 296                                 *t++ = '\b';
 297                                 break;
 298                         case 't':
 299                                 *t++ = '\t';
 300                                 break;
 301                         case 'f':
 302                                 *t++ = '\f';
 303                                 break;
 304                         case '\\':
 305                                 *t++ = '\\';
 306                                 break;
 307                         case '"':
 308                                 *t++ = '"';
 309                                 break;
 310                         case 'u':
 311                                 /* Unicode escape */
 312                                 uval = 0;
 313                                 h ++; /* u character */
 314                                 len --;
 315
 316                                 if (len > 3) {
 317                                         for (i = 0; i < 4; i++) {
 318                                                 uval <<= 4;
 319                                                 if (isdigit (h[i])) {
 320                                                         uval += h[i] - '0';
 321                                                 }
 322                                                 else if (h[i] >= 'a' && h[i] <= 'f') {
 323                                                         uval += h[i] - 'a' + 10;
 324                                                 }
 325                                                 else if (h[i] >= 'A' && h[i] <= 'F') {
 326                                                         uval += h[i] - 'A' + 10;
 327                                                 }
 328                                                 else {
 329                                                         break;
 330                                                 }
 331                                         }
 332
 333                                         /* Encode */
 334                                         if(uval < 0x80) {
 335                                                 t[0] = (char)uval;
 336                                                 t ++;
 337                                         }
 338                                         else if(uval < 0x800) {
 339                                                 t[0] = 0xC0 + ((uval & 0x7C0) >> 6);
 340                                                 t[1] = 0x80 + ((uval & 0x03F));
 341                                                 t += 2;
 342                                         }
 343                                         else if(uval < 0x10000) {
 344                                                 t[0] = 0xE0 + ((uval & 0xF000) >> 12);
 345                                                 t[1] = 0x80 + ((uval & 0x0FC0) >> 6);
 346                                                 t[2] = 0x80 + ((uval & 0x003F));
 347                                                 t += 3;
 348                                         }
 349 #if 0
 350                                         /* It's not actually supported now */
 351                                         else if(uval <= 0x10FFFF) {
 352                                                 t[0] = 0xF0 + ((uval & 0x1C0000) >> 18);
 353                                                 t[1] = 0x80 + ((uval & 0x03F000) >> 12);
 354                                                 t[2] = 0x80 + ((uval & 0x000FC0) >> 6);
 355                                                 t[3] = 0x80 + ((uval & 0x00003F));
 356                                                 t += 4;
 357                                         }
 358 #endif
 359                                         else {
 360                                                 *t++ = '?';
 361                                         }
 362
 363                                         /* Consume 4 characters of source */
 364                                         h += 4;
 365                                         len -= 4;
 366
 367                                         if (len > 0) {
 368                                                 len --; /* for '\' character */
 369                                         }
 370                                         continue;
 371                                 }
 372                                 else {
 373                                         *t++ = 'u';
 374                                 }
 375                                 break;
 376                         default:
 377                                 *t++ = *h;
 378                                 break;
 379                         }
 380                         h ++;
 381                         len --;
 382                 }
 383                 else {
 384                         *t++ = *h++;
 385                 }
 386
 387                 if (len > 0) {
 388                         len --;
 389                 }
 390         }
 391         *t = '\0';
 392
 393         return (t - str);
 394 }
 395
 396 char *
 397 ucl_copy_key_trash (const ucl_object_t *obj)
 398 {
 399         ucl_object_t *deconst;
 400
 401         if (obj == NULL) {
 402                 return NULL;
 403         }
 404         if (obj->trash_stack[UCL_TRASH_KEY] == NULL && obj->key != NULL) {
 405                 deconst = __DECONST (ucl_object_t *, obj);
 406                 deconst->trash_stack[UCL_TRASH_KEY] = malloc (obj->keylen + 1);
 407                 if (deconst->trash_stack[UCL_TRASH_KEY] != NULL) {
 408                         memcpy (deconst->trash_stack[UCL_TRASH_KEY], obj->key, obj->keylen);
 409                         deconst->trash_stack[UCL_TRASH_KEY][obj->keylen] = '\0';
 410                 }
 411                 deconst->key = obj->trash_stack[UCL_TRASH_KEY];
 412                 deconst->flags |= UCL_OBJECT_ALLOCATED_KEY;
 413         }
 414
 415         return obj->trash_stack[UCL_TRASH_KEY];
 416 }
 417
 418 char *
 419 ucl_copy_value_trash (const ucl_object_t *obj)
 420 {
 421         ucl_object_t *deconst;
 422
 423         if (obj == NULL) {
 424                 return NULL;
 425         }
 426         if (obj->trash_stack[UCL_TRASH_VALUE] == NULL) {
 427                 deconst = __DECONST (ucl_object_t *, obj);
 428                 if (obj->type == UCL_STRING) {
 429
 430                         /* Special case for strings */
 431                         if (obj->flags & UCL_OBJECT_BINARY) {
 432                                 deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len);
 433                                 if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) {
 434                                         memcpy (deconst->trash_stack[UCL_TRASH_VALUE],
 435                                                         obj->value.sv,
 436                                                         obj->len);
 437                                         deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE];
 438                                 }
 439                         }
 440                         else {
 441                                 deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len + 1);
 442                                 if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) {
 443                                         memcpy (deconst->trash_stack[UCL_TRASH_VALUE],
 444                                                         obj->value.sv,
 445                                                         obj->len);
 446                                         deconst->trash_stack[UCL_TRASH_VALUE][obj->len] = '\0';
 447                                         deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE];
 448                                 }
 449                         }
 450                 }
 451                 else {
 452                         /* Just emit value in json notation */
 453                         deconst->trash_stack[UCL_TRASH_VALUE] = ucl_object_emit_single_json (obj);
 454                         deconst->len = strlen (obj->trash_stack[UCL_TRASH_VALUE]);
 455                 }
 456                 deconst->flags |= UCL_OBJECT_ALLOCATED_VALUE;
 457         }
 458
 459         return obj->trash_stack[UCL_TRASH_VALUE];
 460 }
 461
 462 ucl_object_t*
 463 ucl_parser_get_object (struct ucl_parser *parser)
 464 {
 465         if (parser->state != UCL_STATE_ERROR && parser->top_obj != NULL) {
 466                 return ucl_object_ref (parser->top_obj);
 467         }
 468
 469         return NULL;
 470 }
 471
 472 void
 473 ucl_parser_free (struct ucl_parser *parser)
 474 {
 475         struct ucl_stack *stack, *stmp;
 476         struct ucl_macro *macro, *mtmp;
 477         struct ucl_chunk *chunk, *ctmp;
 478         struct ucl_pubkey *key, *ktmp;
 479         struct ucl_variable *var, *vtmp;
 480         ucl_object_t *tr, *trtmp;
 481
 482         if (parser == NULL) {
 483                 return;
 484         }
 485
 486         if (parser->top_obj != NULL) {
 487                 ucl_object_unref (parser->top_obj);
 488         }
 489
 490         if (parser->includepaths != NULL) {
 491                 ucl_object_unref (parser->includepaths);
 492         }
 493
 494         LL_FOREACH_SAFE (parser->stack, stack, stmp) {
 495                 free (stack);
 496         }
 497         HASH_ITER (hh, parser->macroes, macro, mtmp) {
 498                 free (macro->name);
 499                 HASH_DEL (parser->macroes, macro);
 500                 UCL_FREE (sizeof (struct ucl_macro), macro);
 501         }
 502         LL_FOREACH_SAFE (parser->chunks, chunk, ctmp) {
 503                 UCL_FREE (sizeof (struct ucl_chunk), chunk);
 504         }
 505         LL_FOREACH_SAFE (parser->keys, key, ktmp) {
 506                 UCL_FREE (sizeof (struct ucl_pubkey), key);
 507         }
 508         LL_FOREACH_SAFE (parser->variables, var, vtmp) {
 509                 free (var->value);
 510                 free (var->var);
 511                 UCL_FREE (sizeof (struct ucl_variable), var);
 512         }
 513         LL_FOREACH_SAFE (parser->trash_objs, tr, trtmp) {
 514                 ucl_object_free_internal (tr, false, ucl_object_dtor_free);
 515         }
 516
 517         if (parser->err != NULL) {
 518                 utstring_free (parser->err);
 519         }
 520
 521         if (parser->cur_file) {
 522                 free (parser->cur_file);
 523         }
 524
 525         if (parser->comments) {
 526                 ucl_object_unref (parser->comments);
 527         }
 528
 529         UCL_FREE (sizeof (struct ucl_parser), parser);
 530 }
 531
 532 const char *
 533 ucl_parser_get_error(struct ucl_parser *parser)
 534 {
 535         if (parser == NULL) {
 536                 return NULL;
 537         }
 538
 539         if (parser->err == NULL) {
 540                 return NULL;
 541         }
 542
 543         return utstring_body (parser->err);
 544 }
 545
 546 int
 547 ucl_parser_get_error_code(struct ucl_parser *parser)
 548 {
 549         if (parser == NULL) {
 550                 return 0;
 551         }
 552
 553         return parser->err_code;
 554 }
 555
 556 unsigned
 557 ucl_parser_get_column(struct ucl_parser *parser)
 558 {
 559         if (parser == NULL || parser->chunks == NULL) {
 560                 return 0;
 561         }
 562
 563         return parser->chunks->column;
 564 }
 565
 566 unsigned
 567 ucl_parser_get_linenum(struct ucl_parser *parser)
 568 {
 569         if (parser == NULL || parser->chunks == NULL) {
 570                 return 0;
 571         }
 572
 573         return parser->chunks->line;
 574 }
 575
 576 void
 577 ucl_parser_clear_error(struct ucl_parser *parser)
 578 {
 579         if (parser != NULL && parser->err != NULL) {
 580                 utstring_free(parser->err);
 581                 parser->err = NULL;
 582                 parser->err_code = 0;
 583         }
 584 }
 585
 586 bool
 587 ucl_pubkey_add (struct ucl_parser *parser, const unsigned char *key, size_t len)
 588 {
 589 #ifndef HAVE_OPENSSL
 590         ucl_create_err (&parser->err, "cannot check signatures without openssl");
 591         return false;
 592 #else
 593 # if (OPENSSL_VERSION_NUMBER < 0x10000000L)
 594         ucl_create_err (&parser->err, "cannot check signatures, openssl version is unsupported");
 595         return EXIT_FAILURE;
 596 # else
 597         struct ucl_pubkey *nkey;
 598         BIO *mem;
 599
 600         mem = BIO_new_mem_buf ((void *)key, len);
 601         nkey = UCL_ALLOC (sizeof (struct ucl_pubkey));
 602         if (nkey == NULL) {
 603                 ucl_create_err (&parser->err, "cannot allocate memory for key");
 604                 return false;
 605         }
 606         nkey->key = PEM_read_bio_PUBKEY (mem, &nkey->key, NULL, NULL);
 607         BIO_free (mem);
 608         if (nkey->key == NULL) {
 609                 UCL_FREE (sizeof (struct ucl_pubkey), nkey);
 610                 ucl_create_err (&parser->err, "%s",
 611                                 ERR_error_string (ERR_get_error (), NULL));
 612                 return false;
 613         }
 614         LL_PREPEND (parser->keys, nkey);
 615 # endif
 616 #endif
 617         return true;
 618 }
 619
 620 #ifdef CURL_FOUND
 621 struct ucl_curl_cbdata {
 622         unsigned char *buf;
 623         size_t buflen;
 624 };
 625
 626 static size_t
 627 ucl_curl_write_callback (void* contents, size_t size, size_t nmemb, void* ud)
 628 {
 629         struct ucl_curl_cbdata *cbdata = ud;
 630         size_t realsize = size * nmemb;
 631
 632         cbdata->buf = realloc (cbdata->buf, cbdata->buflen + realsize + 1);
 633         if (cbdata->buf == NULL) {
 634                 return 0;
 635         }
 636
 637         memcpy (&(cbdata->buf[cbdata->buflen]), contents, realsize);
 638         cbdata->buflen += realsize;
 639         cbdata->buf[cbdata->buflen] = 0;
 640
 641         return realsize;
 642 }
 643 #endif
 644
 645 /**
 646  * Fetch a url and save results to the memory buffer
 647  * @param url url to fetch
 648  * @param len length of url
 649  * @param buf target buffer
 650  * @param buflen target length
 651  * @return
 652  */
 653 bool
 654 ucl_fetch_url (const unsigned char *url, unsigned char **buf, size_t *buflen,
 655                 UT_string **err, bool must_exist)
 656 {
 657
 658 #ifdef HAVE_FETCH_H
 659         struct url *fetch_url;
 660         struct url_stat us;
 661         FILE *in;
 662
 663         fetch_url = fetchParseURL (url);
 664         if (fetch_url == NULL) {
 665                 ucl_create_err (err, "invalid URL %s: %s",
 666                                 url, strerror (errno));
 667                 return false;
 668         }
 669         if ((in = fetchXGet (fetch_url, &us, "")) == NULL) {
 670                 if (!must_exist) {
 671                         ucl_create_err (err, "cannot fetch URL %s: %s",
 672                                 url, strerror (errno));
 673                 }
 674                 fetchFreeURL (fetch_url);
 675                 return false;
 676         }
 677
 678         *buflen = us.size;
 679         *buf = malloc (*buflen);
 680         if (*buf == NULL) {
 681                 ucl_create_err (err, "cannot allocate buffer for URL %s: %s",
 682                                 url, strerror (errno));
 683                 fclose (in);
 684                 fetchFreeURL (fetch_url);
 685                 return false;
 686         }
 687
 688         if (fread (*buf, *buflen, 1, in) != 1) {
 689                 ucl_create_err (err, "cannot read URL %s: %s",
 690                                 url, strerror (errno));
 691                 fclose (in);
 692                 fetchFreeURL (fetch_url);
 693                 return false;
 694         }
 695
 696         fetchFreeURL (fetch_url);
 697         return true;
 698 #elif defined(CURL_FOUND)
 699         CURL *curl;
 700         int r;
 701         struct ucl_curl_cbdata cbdata;
 702
 703         curl = curl_easy_init ();
 704         if (curl == NULL) {
 705                 ucl_create_err (err, "CURL interface is broken");
 706                 return false;
 707         }
 708         if ((r = curl_easy_setopt (curl, CURLOPT_URL, url)) != CURLE_OK) {
 709                 ucl_create_err (err, "invalid URL %s: %s",
 710                                 url, curl_easy_strerror (r));
 711                 curl_easy_cleanup (curl);
 712                 return false;
 713         }
 714         curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, ucl_curl_write_callback);
 715         cbdata.buf = NULL;
 716         cbdata.buflen = 0;
 717         curl_easy_setopt (curl, CURLOPT_WRITEDATA, &cbdata);
 718
 719         if ((r = curl_easy_perform (curl)) != CURLE_OK) {
 720                 if (!must_exist) {
 721                         ucl_create_err (err, "error fetching URL %s: %s",
 722                                 url, curl_easy_strerror (r));
 723                 }
 724                 curl_easy_cleanup (curl);
 725                 if (cbdata.buf) {
 726                         free (cbdata.buf);
 727                 }
 728                 return false;
 729         }
 730         *buf = cbdata.buf;
 731         *buflen = cbdata.buflen;
 732
 733         return true;
 734 #else
 735         ucl_create_err (err, "URL support is disabled");
 736         return false;
 737 #endif
 738 }
 739
 740 /**
 741  * Fetch a file and save results to the memory buffer
 742  * @param filename filename to fetch
 743  * @param len length of filename
 744  * @param buf target buffer
 745  * @param buflen target length
 746  * @return
 747  */
 748 bool
 749 ucl_fetch_file (const unsigned char *filename, unsigned char **buf, size_t *buflen,
 750                 UT_string **err, bool must_exist)
 751 {
 752         int fd;
 753         struct stat st;
 754
 755         if (stat (filename, &st) == -1 || !S_ISREG (st.st_mode)) {
 756                 if (must_exist) {
 757                         ucl_create_err (err, "cannot stat file %s: %s",
 758                                         filename, strerror (errno));
 759                 }
 760                 return false;
 761         }
 762         if (st.st_size == 0) {
 763                 /* Do not map empty files */
 764                 *buf = NULL;
 765                 *buflen = 0;
 766         }
 767         else {
 768                 if ((fd = open (filename, O_RDONLY)) == -1) {
 769                         ucl_create_err (err, "cannot open file %s: %s",
 770                                         filename, strerror (errno));
 771                         return false;
 772                 }
 773                 if ((*buf = ucl_mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
 774                         close (fd);
 775                         ucl_create_err (err, "cannot mmap file %s: %s",
 776                                         filename, strerror (errno));
 777                         *buf = NULL;
 778
 779                         return false;
 780                 }
 781                 *buflen = st.st_size;
 782                 close (fd);
 783         }
 784
 785         return true;
 786 }
 787
 788
 789 #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
 790 static inline bool
 791 ucl_sig_check (const unsigned char *data, size_t datalen,
 792                 const unsigned char *sig, size_t siglen, struct ucl_parser *parser)
 793 {
 794         struct ucl_pubkey *key;
 795         char dig[EVP_MAX_MD_SIZE];
 796         unsigned int diglen;
 797         EVP_PKEY_CTX *key_ctx;
 798         EVP_MD_CTX *sign_ctx = NULL;
 799
 800         sign_ctx = EVP_MD_CTX_create ();
 801
 802         LL_FOREACH (parser->keys, key) {
 803                 key_ctx = EVP_PKEY_CTX_new (key->key, NULL);
 804                 if (key_ctx != NULL) {
 805                         if (EVP_PKEY_verify_init (key_ctx) <= 0) {
 806                                 EVP_PKEY_CTX_free (key_ctx);
 807                                 continue;
 808                         }
 809                         if (EVP_PKEY_CTX_set_rsa_padding (key_ctx, RSA_PKCS1_PADDING) <= 0) {
 810                                 EVP_PKEY_CTX_free (key_ctx);
 811                                 continue;
 812                         }
 813                         if (EVP_PKEY_CTX_set_signature_md (key_ctx, EVP_sha256 ()) <= 0) {
 814                                 EVP_PKEY_CTX_free (key_ctx);
 815                                 continue;
 816                         }
 817                         EVP_DigestInit (sign_ctx, EVP_sha256 ());
 818                         EVP_DigestUpdate (sign_ctx, data, datalen);
 819                         EVP_DigestFinal (sign_ctx, dig, &diglen);
 820
 821                         if (EVP_PKEY_verify (key_ctx, sig, siglen, dig, diglen) == 1) {
 822                                 EVP_MD_CTX_destroy (sign_ctx);
 823                                 EVP_PKEY_CTX_free (key_ctx);
 824                                 return true;
 825                         }
 826
 827                         EVP_PKEY_CTX_free (key_ctx);
 828                 }
 829         }
 830
 831         EVP_MD_CTX_destroy (sign_ctx);
 832
 833         return false;
 834 }
 835 #endif
 836
 837 struct ucl_include_params {
 838         bool check_signature;
 839         bool must_exist;
 840         bool use_glob;
 841         bool use_prefix;
 842         bool soft_fail;
 843         bool allow_glob;
 844         unsigned priority;
 845         enum ucl_duplicate_strategy strat;
 846         enum ucl_parse_type parse_type;
 847         const char *prefix;
 848         const char *target;
 849 };
 850
 851 /**
 852  * Include an url to configuration
 853  * @param data
 854  * @param len
 855  * @param parser
 856  * @param err
 857  * @return
 858  */
 859 static bool
 860 ucl_include_url (const unsigned char *data, size_t len,
 861                 struct ucl_parser *parser,
 862                 struct ucl_include_params *params)
 863 {
 864
 865         bool res;
 866         unsigned char *buf = NULL;
 867         size_t buflen = 0;
 868         struct ucl_chunk *chunk;
 869         char urlbuf[PATH_MAX];
 870         int prev_state;
 871
 872         snprintf (urlbuf, sizeof (urlbuf), "%.*s", (int)len, data);
 873
 874         if (!ucl_fetch_url (urlbuf, &buf, &buflen, &parser->err, params->must_exist)) {
 875                 return !params->must_exist;
 876         }
 877
 878         if (params->check_signature) {
 879 #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
 880                 unsigned char *sigbuf = NULL;
 881                 size_t siglen = 0;
 882                 /* We need to check signature first */
 883                 snprintf (urlbuf, sizeof (urlbuf), "%.*s.sig", (int)len, data);
 884                 if (!ucl_fetch_url (urlbuf, &sigbuf, &siglen, &parser->err, true)) {
 885                         return false;
 886                 }
 887                 if (!ucl_sig_check (buf, buflen, sigbuf, siglen, parser)) {
 888                         ucl_create_err (&parser->err, "cannot verify url %s: %s",
 889                                                         urlbuf,
 890                                                         ERR_error_string (ERR_get_error (), NULL));
 891                         if (siglen > 0) {
 892                                 ucl_munmap (sigbuf, siglen);
 893                         }
 894                         return false;
 895                 }
 896                 if (siglen > 0) {
 897                         ucl_munmap (sigbuf, siglen);
 898                 }
 899 #endif
 900         }
 901
 902         prev_state = parser->state;
 903         parser->state = UCL_STATE_INIT;
 904
 905         res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority,
 906                         params->strat, params->parse_type);
 907         if (res == true) {
 908                 /* Remove chunk from the stack */
 909                 chunk = parser->chunks;
 910                 if (chunk != NULL) {
 911                         parser->chunks = chunk->next;
 912                         UCL_FREE (sizeof (struct ucl_chunk), chunk);
 913                 }
 914         }
 915
 916         parser->state = prev_state;
 917         free (buf);
 918
 919         return res;
 920 }
 921
 922 /**
 923  * Include a single file to the parser
 924  * @param data
 925  * @param len
 926  * @param parser
 927  * @param check_signature
 928  * @param must_exist
 929  * @param allow_glob
 930  * @param priority
 931  * @return
 932  */
 933 static bool
 934 ucl_include_file_single (const unsigned char *data, size_t len,
 935                 struct ucl_parser *parser, struct ucl_include_params *params)
 936 {
 937         bool res;
 938         struct ucl_chunk *chunk;
 939         unsigned char *buf = NULL;
 940         char *old_curfile, *ext;
 941         size_t buflen = 0;
 942         char filebuf[PATH_MAX], realbuf[PATH_MAX];
 943         int prev_state;
 944         struct ucl_variable *cur_var, *tmp_var, *old_curdir = NULL,
 945                         *old_filename = NULL;
 946         ucl_object_t *nest_obj = NULL, *old_obj = NULL, *new_obj = NULL;
 947         ucl_hash_t *container = NULL;
 948         struct ucl_stack *st = NULL;
 949
 950         snprintf (filebuf, sizeof (filebuf), "%.*s", (int)len, data);
 951         if (ucl_realpath (filebuf, realbuf) == NULL) {
 952                 if (params->soft_fail) {
 953                         return false;
 954                 }
 955                 if (!params->must_exist) {
 956                         return true;
 957                 }
 958                 ucl_create_err (&parser->err, "cannot open file %s: %s",
 959                                                                         filebuf,
 960                                                                         strerror (errno));
 961                 return false;
 962         }
 963
 964         if (parser->cur_file && strcmp (realbuf, parser->cur_file) == 0) {
 965                 /* We are likely including the file itself */
 966                 if (params->soft_fail) {
 967                         return false;
 968                 }
 969
 970                 ucl_create_err (&parser->err, "trying to include the file %s from itself",
 971                                 realbuf);
 972                 return false;
 973         }
 974
 975         if (!ucl_fetch_file (realbuf, &buf, &buflen, &parser->err, params->must_exist)) {
 976                 if (params->soft_fail) {
 977                         return false;
 978                 }
 979
 980                 return (!params->must_exist || false);
 981         }
 982
 983         if (params->check_signature) {
 984 #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
 985                 unsigned char *sigbuf = NULL;
 986                 size_t siglen = 0;
 987                 /* We need to check signature first */
 988                 snprintf (filebuf, sizeof (filebuf), "%s.sig", realbuf);
 989                 if (!ucl_fetch_file (filebuf, &sigbuf, &siglen, &parser->err, true)) {
 990                         return false;
 991                 }
 992                 if (!ucl_sig_check (buf, buflen, sigbuf, siglen, parser)) {
 993                         ucl_create_err (&parser->err, "cannot verify file %s: %s",
 994                                                         filebuf,
 995                                                         ERR_error_string (ERR_get_error (), NULL));
 996                         if (sigbuf) {
 997                                 ucl_munmap (sigbuf, siglen);
 998                         }
 999                         return false;
1000                 }
1001                 if (sigbuf) {
1002                         ucl_munmap (sigbuf, siglen);
1003                 }
1004 #endif
1005         }
1006
1007         old_curfile = parser->cur_file;
1008         parser->cur_file = strdup (realbuf);
1009
1010         /* Store old file vars */
1011         DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) {
1012                 if (strcmp (cur_var->var, "CURDIR") == 0) {
1013                         old_curdir = cur_var;
1014                         DL_DELETE (parser->variables, cur_var);
1015                 }
1016                 else if (strcmp (cur_var->var, "FILENAME") == 0) {
1017                         old_filename = cur_var;
1018                         DL_DELETE (parser->variables, cur_var);
1019                 }
1020         }
1021
1022         ucl_parser_set_filevars (parser, realbuf, false);
1023
1024         prev_state = parser->state;
1025         parser->state = UCL_STATE_INIT;
1026
1027         if (params->use_prefix && params->prefix == NULL) {
1028                 /* Auto generate a key name based on the included filename */
1029                 params->prefix = basename (realbuf);
1030                 ext = strrchr (params->prefix, '.');
1031                 if (ext != NULL && (strcmp (ext, ".conf") == 0 || strcmp (ext, ".ucl") == 0)) {
1032                         /* Strip off .conf or .ucl */
1033                         *ext = '\0';
1034                 }
1035         }
1036         if (params->prefix != NULL) {
1037                 /* This is a prefixed include */
1038                 container = parser->stack->obj->value.ov;
1039
1040                 old_obj = __DECONST (ucl_object_t *, ucl_hash_search (container,
1041                                 params->prefix, strlen (params->prefix)));
1042
1043                 if (strcasecmp (params->target, "array") == 0 && old_obj == NULL) {
1044                         /* Create an array with key: prefix */
1045                         old_obj = ucl_object_new_full (UCL_ARRAY, params->priority);
1046                         old_obj->key = params->prefix;
1047                         old_obj->keylen = strlen (params->prefix);
1048                         ucl_copy_key_trash(old_obj);
1049                         old_obj->prev = old_obj;
1050                         old_obj->next = NULL;
1051
1052                         container = ucl_hash_insert_object (container, old_obj,
1053                                         parser->flags & UCL_PARSER_KEY_LOWERCASE);
1054                         parser->stack->obj->len ++;
1055
1056                         nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
1057                         nest_obj->prev = nest_obj;
1058                         nest_obj->next = NULL;
1059
1060                         ucl_array_append (old_obj, nest_obj);
1061                 }
1062                 else if (old_obj == NULL) {
1063                         /* Create an object with key: prefix */
1064                         nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
1065
1066                         if (nest_obj == NULL) {
1067                                 ucl_create_err (&parser->err, "cannot allocate memory for an object");
1068                                 if (buf) {
1069                                         ucl_munmap (buf, buflen);
1070                                 }
1071
1072                                 return false;
1073                         }
1074
1075                         nest_obj->key = params->prefix;
1076                         nest_obj->keylen = strlen (params->prefix);
1077                         ucl_copy_key_trash(nest_obj);
1078                         nest_obj->prev = nest_obj;
1079                         nest_obj->next = NULL;
1080
1081                         container = ucl_hash_insert_object (container, nest_obj,
1082                                         parser->flags & UCL_PARSER_KEY_LOWERCASE);
1083                         parser->stack->obj->len ++;
1084                 }
1085                 else if (strcasecmp (params->target, "array") == 0 ||
1086                                 ucl_object_type(old_obj) == UCL_ARRAY) {
1087                         if (ucl_object_type(old_obj) == UCL_ARRAY) {
1088                                 /* Append to the existing array */
1089                                 nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
1090                                 if (nest_obj == NULL) {
1091                                         ucl_create_err (&parser->err, "cannot allocate memory for an object");
1092                                         if (buf) {
1093                                                 ucl_munmap (buf, buflen);
1094                                         }
1095
1096                                         return false;
1097                                 }
1098                                 nest_obj->prev = nest_obj;
1099                                 nest_obj->next = NULL;
1100
1101                                 ucl_array_append (old_obj, nest_obj);
1102                         }
1103                         else {
1104                                 /* Convert the object to an array */
1105                                 new_obj = ucl_object_typed_new (UCL_ARRAY);
1106                                 if (new_obj == NULL) {
1107                                         ucl_create_err (&parser->err, "cannot allocate memory for an object");
1108                                         if (buf) {
1109                                                 ucl_munmap (buf, buflen);
1110                                         }
1111
1112                                         return false;
1113                                 }
1114                                 new_obj->key = old_obj->key;
1115                                 new_obj->keylen = old_obj->keylen;
1116                                 new_obj->flags |= UCL_OBJECT_MULTIVALUE;
1117                                 new_obj->prev = new_obj;
1118                                 new_obj->next = NULL;
1119
1120                                 nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
1121                                 if (nest_obj == NULL) {
1122                                         ucl_create_err (&parser->err, "cannot allocate memory for an object");
1123                                         if (buf) {
1124                                                 ucl_munmap (buf, buflen);
1125                                         }
1126
1127                                         return false;
1128                                 }
1129                                 nest_obj->prev = nest_obj;
1130                                 nest_obj->next = NULL;
1131
1132                                 ucl_array_append (new_obj, old_obj);
1133                                 ucl_array_append (new_obj, nest_obj);
1134                                 ucl_hash_replace (container, old_obj, new_obj);
1135                         }
1136                 }
1137                 else {
1138                         if (ucl_object_type (old_obj) == UCL_OBJECT) {
1139                                 /* Append to existing Object*/
1140                                 nest_obj = old_obj;
1141                         }
1142                         else {
1143                                 /* The key is not an object */
1144                                 ucl_create_err (&parser->err,
1145                                                 "Conflicting type for key: %s",
1146                                                 params->prefix);
1147                                 if (buf) {
1148                                         ucl_munmap (buf, buflen);
1149                                 }
1150
1151                                 return false;
1152                         }
1153                 }
1154
1155                  /* Put all of the content of the include inside that object */
1156                 parser->stack->obj->value.ov = container;
1157
1158                 st = UCL_ALLOC (sizeof (struct ucl_stack));
1159                 if (st == NULL) {
1160                         ucl_create_err (&parser->err, "cannot allocate memory for an object");
1161                         ucl_object_unref (nest_obj);
1162
1163                         if (buf) {
1164                                 ucl_munmap (buf, buflen);
1165                         }
1166
1167                         return false;
1168                 }
1169                 st->obj = nest_obj;
1170                 st->level = parser->stack->level;
1171                 LL_PREPEND (parser->stack, st);
1172                 parser->cur_obj = nest_obj;
1173         }
1174
1175         res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority,
1176                         params->strat, params->parse_type);
1177
1178         if (!res) {
1179                 if (!params->must_exist) {
1180                         /* Free error */
1181                         utstring_free (parser->err);
1182                         parser->err = NULL;
1183                         res = true;
1184                 }
1185         }
1186
1187         /* Stop nesting the include, take 1 level off the stack */
1188         if (params->prefix != NULL && nest_obj != NULL) {
1189                 parser->stack = st->next;
1190                 UCL_FREE (sizeof (struct ucl_stack), st);
1191         }
1192
1193         /* Remove chunk from the stack */
1194         chunk = parser->chunks;
1195         if (chunk != NULL) {
1196                 parser->chunks = chunk->next;
1197                 UCL_FREE (sizeof (struct ucl_chunk), chunk);
1198                 parser->recursion --;
1199         }
1200
1201         /* Restore old file vars */
1202         if (parser->cur_file) {
1203                 free (parser->cur_file);
1204         }
1205
1206         parser->cur_file = old_curfile;
1207         DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) {
1208                 if (strcmp (cur_var->var, "CURDIR") == 0 && old_curdir) {
1209                         DL_DELETE (parser->variables, cur_var);
1210                         free (cur_var->var);
1211                         free (cur_var->value);
1212                         UCL_FREE (sizeof (struct ucl_variable), cur_var);
1213                 }
1214                 else if (strcmp (cur_var->var, "FILENAME") == 0 && old_filename) {
1215                         DL_DELETE (parser->variables, cur_var);
1216                         free (cur_var->var);
1217                         free (cur_var->value);
1218                         UCL_FREE (sizeof (struct ucl_variable), cur_var);
1219                 }
1220         }
1221         if (old_filename) {
1222                 DL_APPEND (parser->variables, old_filename);
1223         }
1224         if (old_curdir) {
1225                 DL_APPEND (parser->variables, old_curdir);
1226         }
1227
1228         parser->state = prev_state;
1229
1230         if (buflen > 0) {
1231                 ucl_munmap (buf, buflen);
1232         }
1233
1234         return res;
1235 }
1236
1237 /**
1238  * Include a file to configuration
1239  * @param data
1240  * @param len
1241  * @param parser
1242  * @param err
1243  * @return
1244  */
1245 static bool
1246 ucl_include_file (const unsigned char *data, size_t len,
1247                 struct ucl_parser *parser, struct ucl_include_params *params)
1248 {
1249         const unsigned char *p = data, *end = data + len;
1250         bool need_glob = false;
1251         int cnt = 0;
1252         char glob_pattern[PATH_MAX];
1253         size_t i;
1254
1255 #ifndef _WIN32
1256         if (!params->allow_glob) {
1257                 return ucl_include_file_single (data, len, parser, params);
1258         }
1259         else {
1260                 /* Check for special symbols in a filename */
1261                 while (p != end) {
1262                         if (*p == '*' || *p == '?') {
1263                                 need_glob = true;
1264                                 break;
1265                         }
1266                         p ++;
1267                 }
1268                 if (need_glob) {
1269                         glob_t globbuf;
1270                         memset (&globbuf, 0, sizeof (globbuf));
1271                         ucl_strlcpy (glob_pattern, (const char *)data,
1272                                 (len + 1 < sizeof (glob_pattern) ? len + 1 : sizeof (glob_pattern)));
1273                         if (glob (glob_pattern, 0, NULL, &globbuf) != 0) {
1274                                 return (!params->must_exist || false);
1275                         }
1276                         for (i = 0; i < globbuf.gl_pathc; i ++) {
1277                                 if (!ucl_include_file_single ((unsigned char *)globbuf.gl_pathv[i],
1278                                                 strlen (globbuf.gl_pathv[i]), parser, params)) {
1279                                         if (params->soft_fail) {
1280                                                 continue;
1281                                         }
1282                                         globfree (&globbuf);
1283                                         return false;
1284                                 }
1285                                 cnt ++;
1286                         }
1287                         globfree (&globbuf);
1288
1289                         if (cnt == 0 && params->must_exist) {
1290                                 ucl_create_err (&parser->err, "cannot match any files for pattern %s",
1291                                         glob_pattern);
1292                                 return false;
1293                         }
1294                 }
1295                 else {
1296                         return ucl_include_file_single (data, len, parser, params);
1297                 }
1298         }
1299 #else
1300         /* Win32 compilers do not support globbing. Therefore, for Win32,
1301            treat allow_glob/need_glob as a NOOP and just return */
1302         return ucl_include_file_single (data, len, parser, params);
1303 #endif
1304
1305         return true;
1306 }
1307
1308 /**
1309  * Common function to handle .*include* macros
1310  * @param data
1311  * @param len
1312  * @param args
1313  * @param parser
1314  * @param default_try
1315  * @param default_sign
1316  * @return
1317  */
1318 static bool
1319 ucl_include_common (const unsigned char *data, size_t len,
1320                 const ucl_object_t *args, struct ucl_parser *parser,
1321                 bool default_try,
1322                 bool default_sign)
1323 {
1324         bool allow_url = false, search = false;
1325         const char *duplicate;
1326         const ucl_object_t *param;
1327         ucl_object_iter_t it = NULL, ip = NULL;
1328         char ipath[PATH_MAX];
1329         struct ucl_include_params params;
1330
1331         /* Default values */
1332         params.soft_fail = default_try;
1333         params.allow_glob = false;
1334         params.check_signature = default_sign;
1335         params.use_prefix = false;
1336         params.target = "object";
1337         params.prefix = NULL;
1338         params.priority = 0;
1339         params.parse_type = UCL_PARSE_UCL;
1340         params.strat = UCL_DUPLICATE_APPEND;
1341         params.must_exist = !default_try;
1342
1343         /* Process arguments */
1344         if (args != NULL && args->type == UCL_OBJECT) {
1345                 while ((param = ucl_object_iterate (args, &it, true)) != NULL) {
1346                         if (param->type == UCL_BOOLEAN) {
1347                                 if (strncmp (param->key, "try", param->keylen) == 0) {
1348                                         params.must_exist = !ucl_object_toboolean (param);
1349                                 }
1350                                 else if (strncmp (param->key, "sign", param->keylen) == 0) {
1351                                         params.check_signature = ucl_object_toboolean (param);
1352                                 }
1353                                 else if (strncmp (param->key, "glob", param->keylen) == 0) {
1354                                         params.allow_glob = ucl_object_toboolean (param);
1355                                 }
1356                                 else if (strncmp (param->key, "url", param->keylen) == 0) {
1357                                         allow_url = ucl_object_toboolean (param);
1358                                 }
1359                                 else if (strncmp (param->key, "prefix", param->keylen) == 0) {
1360                                         params.use_prefix = ucl_object_toboolean (param);
1361                                 }
1362                         }
1363                         else if (param->type == UCL_STRING) {
1364                                 if (strncmp (param->key, "key", param->keylen) == 0) {
1365                                         params.prefix = ucl_object_tostring (param);
1366                                 }
1367                                 else if (strncmp (param->key, "target", param->keylen) == 0) {
1368                                         params.target = ucl_object_tostring (param);
1369                                 }
1370                                 else if (strncmp (param->key, "duplicate", param->keylen) == 0) {
1371                                         duplicate = ucl_object_tostring (param);
1372
1373                                         if (strcmp (duplicate, "append") == 0) {
1374                                                 params.strat = UCL_DUPLICATE_APPEND;
1375                                         }
1376                                         else if (strcmp (duplicate, "merge") == 0) {
1377                                                 params.strat = UCL_DUPLICATE_MERGE;
1378                                         }
1379                                         else if (strcmp (duplicate, "rewrite") == 0) {
1380                                                 params.strat = UCL_DUPLICATE_REWRITE;
1381                                         }
1382                                         else if (strcmp (duplicate, "error") == 0) {
1383                                                 params.strat = UCL_DUPLICATE_ERROR;
1384                                         }
1385                                 }
1386                         }
1387                         else if (param->type == UCL_ARRAY) {
1388                                 if (strncmp (param->key, "path", param->keylen) == 0) {
1389                                         ucl_set_include_path (parser, __DECONST(ucl_object_t *, param));
1390                                 }
1391                         }
1392                         else if (param->type == UCL_INT) {
1393                                 if (strncmp (param->key, "priority", param->keylen) == 0) {
1394                                         params.priority = ucl_object_toint (param);
1395                                 }
1396                         }
1397                 }
1398         }
1399
1400         if (parser->includepaths == NULL) {
1401                 if (allow_url && ucl_strnstr (data, "://", len) != NULL) {
1402                         /* Globbing is not used for URL's */
1403                         return ucl_include_url (data, len, parser, &params);
1404                 }
1405                 else if (data != NULL) {
1406                         /* Try to load a file */
1407                         return ucl_include_file (data, len, parser, &params);
1408                 }
1409         }
1410         else {
1411                 if (allow_url && ucl_strnstr (data, "://", len) != NULL) {
1412                         /* Globbing is not used for URL's */
1413                         return ucl_include_url (data, len, parser, &params);
1414                 }
1415
1416                 ip = ucl_object_iterate_new (parser->includepaths);
1417                 while ((param = ucl_object_iterate_safe (ip, true)) != NULL) {
1418                         if (ucl_object_type(param) == UCL_STRING) {
1419                                 snprintf (ipath, sizeof (ipath), "%s/%.*s", ucl_object_tostring(param),
1420                                                 (int)len, data);
1421                                 if ((search = ucl_include_file (ipath, strlen (ipath),
1422                                                 parser, &params))) {
1423                                         if (!params.allow_glob) {
1424                                                 break;
1425                                         }
1426                                 }
1427                         }
1428                 }
1429                 ucl_object_iterate_free (ip);
1430                 if (search == true) {
1431                         return true;
1432                 }
1433                 else {
1434                         ucl_create_err (&parser->err,
1435                                         "cannot find file: %.*s in search path",
1436                                         (int)len, data);
1437                         return false;
1438                 }
1439         }
1440
1441         return false;
1442 }
1443
1444 /**
1445  * Handle include macro
1446  * @param data include data
1447  * @param len length of data
1448  * @param args UCL object representing arguments to the macro
1449  * @param ud user data
1450  * @return
1451  */
1452 bool
1453 ucl_include_handler (const unsigned char *data, size_t len,
1454                 const ucl_object_t *args, void* ud)
1455 {
1456         struct ucl_parser *parser = ud;
1457
1458         return ucl_include_common (data, len, args, parser, false, false);
1459 }
1460
1461 /**
1462  * Handle includes macro
1463  * @param data include data
1464  * @param len length of data
1465  * @param args UCL object representing arguments to the macro
1466  * @param ud user data
1467  * @return
1468  */
1469 bool
1470 ucl_includes_handler (const unsigned char *data, size_t len,
1471                 const ucl_object_t *args, void* ud)
1472 {
1473         struct ucl_parser *parser = ud;
1474
1475         return ucl_include_common (data, len, args, parser, false, true);
1476 }
1477
1478 /**
1479  * Handle tryinclude macro
1480  * @param data include data
1481  * @param len length of data
1482  * @param args UCL object representing arguments to the macro
1483  * @param ud user data
1484  * @return
1485  */
1486 bool
1487 ucl_try_include_handler (const unsigned char *data, size_t len,
1488                 const ucl_object_t *args, void* ud)
1489 {
1490         struct ucl_parser *parser = ud;
1491
1492         return ucl_include_common (data, len, args, parser, true, false);
1493 }
1494
1495 /**
1496  * Handle priority macro
1497  * @param data include data
1498  * @param len length of data
1499  * @param args UCL object representing arguments to the macro
1500  * @param ud user data
1501  * @return
1502  */
1503 bool
1504 ucl_priority_handler (const unsigned char *data, size_t len,
1505                 const ucl_object_t *args, void* ud)
1506 {
1507         struct ucl_parser *parser = ud;
1508         unsigned priority = 255;
1509         const ucl_object_t *param;
1510         bool found = false;
1511         char *value = NULL, *leftover = NULL;
1512         ucl_object_iter_t it = NULL;
1513
1514         if (parser == NULL) {
1515                 return false;
1516         }
1517
1518         /* Process arguments */
1519         if (args != NULL && args->type == UCL_OBJECT) {
1520                 while ((param = ucl_object_iterate (args, &it, true)) != NULL) {
1521                         if (param->type == UCL_INT) {
1522                                 if (strncmp (param->key, "priority", param->keylen) == 0) {
1523                                         priority = ucl_object_toint (param);
1524                                         found = true;
1525                                 }
1526                         }
1527                 }
1528         }
1529
1530         if (len > 0) {
1531                 value = malloc(len + 1);
1532                 ucl_strlcpy(value, (const char *)data, len + 1);
1533                 priority = strtol(value, &leftover, 10);
1534                 if (*leftover != '\0') {
1535                         ucl_create_err (&parser->err, "Invalid priority value in macro: %s",
1536                                 value);
1537                         free(value);
1538                         return false;
1539                 }
1540                 free(value);
1541                 found = true;
1542         }
1543
1544         if (found == true) {
1545                 parser->chunks->priority = priority;
1546                 return true;
1547         }
1548
1549         ucl_create_err (&parser->err, "Unable to parse priority macro");
1550         return false;
1551 }
1552
1553 /**
1554  * Handle load macro
1555  * @param data include data
1556  * @param len length of data
1557  * @param args UCL object representing arguments to the macro
1558  * @param ud user data
1559  * @return
1560  */
1561 bool
1562 ucl_load_handler (const unsigned char *data, size_t len,
1563                 const ucl_object_t *args, void* ud)
1564 {
1565         struct ucl_parser *parser = ud;
1566         const ucl_object_t *param;
1567         ucl_object_t *obj, *old_obj;
1568         ucl_object_iter_t it = NULL;
1569         bool try_load, multiline, test;
1570         const char *target, *prefix;
1571         char *load_file, *tmp;
1572         unsigned char *buf;
1573         size_t buflen;
1574         unsigned priority;
1575         int64_t iv;
1576         ucl_object_t *container = NULL;
1577         enum ucl_string_flags flags;
1578
1579         /* Default values */
1580         try_load = false;
1581         multiline = false;
1582         test = false;
1583         target = "string";
1584         prefix = NULL;
1585         load_file = NULL;
1586         buf = NULL;
1587         buflen = 0;
1588         priority = 0;
1589         obj = NULL;
1590         old_obj = NULL;
1591         flags = 0;
1592
1593         if (parser == NULL) {
1594                 return false;
1595         }
1596
1597         /* Process arguments */
1598         if (args != NULL && args->type == UCL_OBJECT) {
1599                 while ((param = ucl_object_iterate (args, &it, true)) != NULL) {
1600                         if (param->type == UCL_BOOLEAN) {
1601                                 if (strncmp (param->key, "try", param->keylen) == 0) {
1602                                         try_load = ucl_object_toboolean (param);
1603                                 }
1604                                 else if (strncmp (param->key, "multiline", param->keylen) == 0) {
1605                                         multiline = ucl_object_toboolean (param);
1606                                 }
1607                                 else if (strncmp (param->key, "escape", param->keylen) == 0) {
1608                                         test = ucl_object_toboolean (param);
1609                                         if (test) {
1610                                                 flags |= UCL_STRING_ESCAPE;
1611                                         }
1612                                 }
1613                                 else if (strncmp (param->key, "trim", param->keylen) == 0) {
1614                                         test = ucl_object_toboolean (param);
1615                                         if (test) {
1616                                                 flags |= UCL_STRING_TRIM;
1617                                         }
1618                                 }
1619                         }
1620                         else if (param->type == UCL_STRING) {
1621                                 if (strncmp (param->key, "key", param->keylen) == 0) {
1622                                         prefix = ucl_object_tostring (param);
1623                                 }
1624                                 else if (strncmp (param->key, "target", param->keylen) == 0) {
1625                                         target = ucl_object_tostring (param);
1626                                 }
1627                         }
1628                         else if (param->type == UCL_INT) {
1629                                 if (strncmp (param->key, "priority", param->keylen) == 0) {
1630                                         priority = ucl_object_toint (param);
1631                                 }
1632                         }
1633                 }
1634         }
1635
1636         if (prefix == NULL || strlen (prefix) == 0) {
1637                 ucl_create_err (&parser->err, "No Key specified in load macro");
1638                 return false;
1639         }
1640
1641         if (len > 0) {
1642                 load_file = malloc (len + 1);
1643                 if (!load_file) {
1644                         ucl_create_err (&parser->err, "cannot allocate memory for suffix");
1645
1646                         return false;
1647                 }
1648
1649                 snprintf (load_file, len + 1, "%.*s", (int)len, data);
1650
1651                 if (!ucl_fetch_file (load_file, &buf, &buflen, &parser->err,
1652                                 !try_load)) {
1653                         free (load_file);
1654
1655                         return (try_load || false);
1656                 }
1657
1658                 free (load_file);
1659                 container = parser->stack->obj;
1660                 old_obj = __DECONST (ucl_object_t *, ucl_object_lookup (container,
1661                                 prefix));
1662
1663                 if (old_obj != NULL) {
1664                         ucl_create_err (&parser->err, "Key %s already exists", prefix);
1665                         if (buf) {
1666                                 ucl_munmap (buf, buflen);
1667                         }
1668
1669                         return false;
1670                 }
1671
1672                 if (strcasecmp (target, "string") == 0) {
1673                         obj = ucl_object_fromstring_common (buf, buflen, flags);
1674                         ucl_copy_value_trash (obj);
1675                         if (multiline) {
1676                                 obj->flags |= UCL_OBJECT_MULTILINE;
1677                         }
1678                 }
1679                 else if (strcasecmp (target, "int") == 0) {
1680                         tmp = malloc (buflen + 1);
1681
1682                         if (tmp == NULL) {
1683                                 ucl_create_err (&parser->err, "Memory allocation failed");
1684                                 if (buf) {
1685                                         ucl_munmap (buf, buflen);
1686                                 }
1687
1688                                 return false;
1689                         }
1690
1691                         snprintf (tmp, buflen + 1, "%.*s", (int)buflen, buf);
1692                         iv = strtoll (tmp, NULL, 10);
1693                         obj = ucl_object_fromint (iv);
1694                         free (tmp);
1695                 }
1696
1697                 if (buf) {
1698                         ucl_munmap (buf, buflen);
1699                 }
1700
1701                 if (obj != NULL) {
1702                         obj->key = prefix;
1703                         obj->keylen = strlen (prefix);
1704                         ucl_copy_key_trash (obj);
1705                         obj->prev = obj;
1706                         obj->next = NULL;
1707                         ucl_object_set_priority (obj, priority);
1708                         ucl_object_insert_key (container, obj, obj->key, obj->keylen, false);
1709                 }
1710
1711                 return true;
1712         }
1713
1714         ucl_create_err (&parser->err, "Unable to parse load macro");
1715         return false;
1716 }
1717
1718 bool
1719 ucl_inherit_handler (const unsigned char *data, size_t len,
1720                 const ucl_object_t *args, const ucl_object_t *ctx, void* ud)
1721 {
1722         const ucl_object_t *parent, *cur;
1723         ucl_object_t *target, *copy;
1724         ucl_object_iter_t it = NULL;
1725         bool replace = false;
1726         struct ucl_parser *parser = ud;
1727
1728         parent = ucl_object_lookup_len (ctx, data, len);
1729
1730         /* Some sanity checks */
1731         if (parent == NULL || ucl_object_type (parent) != UCL_OBJECT) {
1732                 ucl_create_err (&parser->err, "Unable to find inherited object %*.s",
1733                                 (int)len, data);
1734                 return false;
1735         }
1736
1737         if (parser->stack == NULL || parser->stack->obj == NULL ||
1738                         ucl_object_type (parser->stack->obj) != UCL_OBJECT) {
1739                 ucl_create_err (&parser->err, "Invalid inherit context");
1740                 return false;
1741         }
1742
1743         target = parser->stack->obj;
1744
1745         if (args && (cur = ucl_object_lookup (args, "replace")) != NULL) {
1746                 replace = ucl_object_toboolean (cur);
1747         }
1748
1749         while ((cur = ucl_object_iterate (parent, &it, true))) {
1750                 /* We do not replace existing keys */
1751                 if (!replace && ucl_object_lookup_len (target, cur->key, cur->keylen)) {
1752                         continue;
1753                 }
1754
1755                 copy = ucl_object_copy (cur);
1756
1757                 if (!replace) {
1758                         copy->flags |= UCL_OBJECT_INHERITED;
1759                 }
1760
1761                 ucl_object_insert_key (target, copy, copy->key,
1762                                 copy->keylen, false);
1763         }
1764
1765         return true;
1766 }
1767
1768 bool
1769 ucl_parser_set_filevars (struct ucl_parser *parser, const char *filename, bool need_expand)
1770 {
1771         char realbuf[PATH_MAX], *curdir;
1772
1773         if (filename != NULL) {
1774                 if (need_expand) {
1775                         if (ucl_realpath (filename, realbuf) == NULL) {
1776                                 return false;
1777                         }
1778                 }
1779                 else {
1780                         ucl_strlcpy (realbuf, filename, sizeof (realbuf));
1781                 }
1782
1783                 /* Define variables */
1784                 ucl_parser_register_variable (parser, "FILENAME", realbuf);
1785                 curdir = dirname (realbuf);
1786                 ucl_parser_register_variable (parser, "CURDIR", curdir);
1787         }
1788         else {
1789                 /* Set everything from the current dir */
1790                 curdir = getcwd (realbuf, sizeof (realbuf));
1791                 ucl_parser_register_variable (parser, "FILENAME", "undef");
1792                 ucl_parser_register_variable (parser, "CURDIR", curdir);
1793         }
1794
1795         return true;
1796 }
1797
1798 bool
1799 ucl_parser_add_file_full (struct ucl_parser *parser, const char *filename,
1800                 unsigned priority, enum ucl_duplicate_strategy strat,
1801                 enum ucl_parse_type parse_type)
1802 {
1803         unsigned char *buf;
1804         size_t len;
1805         bool ret;
1806         char realbuf[PATH_MAX];
1807
1808         if (ucl_realpath (filename, realbuf) == NULL) {
1809                 ucl_create_err (&parser->err, "cannot open file %s: %s",
1810                                 filename,
1811                                 strerror (errno));
1812                 return false;
1813         }
1814
1815         if (!ucl_fetch_file (realbuf, &buf, &len, &parser->err, true)) {
1816                 return false;
1817         }
1818
1819         if (parser->cur_file) {
1820                 free (parser->cur_file);
1821         }
1822         parser->cur_file = strdup (realbuf);
1823         ucl_parser_set_filevars (parser, realbuf, false);
1824         ret = ucl_parser_add_chunk_full (parser, buf, len, priority, strat,
1825                         parse_type);
1826
1827         if (len > 0) {
1828                 ucl_munmap (buf, len);
1829         }
1830
1831         return ret;
1832 }
1833
1834 bool
1835 ucl_parser_add_file_priority (struct ucl_parser *parser, const char *filename,
1836                 unsigned priority)
1837 {
1838         if (parser == NULL) {
1839                 return false;
1840         }
1841
1842         return ucl_parser_add_file_full(parser, filename, priority,
1843                         UCL_DUPLICATE_APPEND, UCL_PARSE_UCL);
1844 }
1845
1846 bool
1847 ucl_parser_add_file (struct ucl_parser *parser, const char *filename)
1848 {
1849         if (parser == NULL) {
1850                 return false;
1851         }
1852
1853         return ucl_parser_add_file_full(parser, filename,
1854                         parser->default_priority, UCL_DUPLICATE_APPEND,
1855                         UCL_PARSE_UCL);
1856 }
1857
1858
1859 bool
1860 ucl_parser_add_fd_full (struct ucl_parser *parser, int fd,
1861                 unsigned priority, enum ucl_duplicate_strategy strat,
1862                 enum ucl_parse_type parse_type)
1863 {
1864         unsigned char *buf;
1865         size_t len;
1866         bool ret;
1867         struct stat st;
1868
1869         if (fstat (fd, &st) == -1) {
1870                 ucl_create_err (&parser->err, "cannot stat fd %d: %s",
1871                         fd, strerror (errno));
1872                 return false;
1873         }
1874         if (st.st_size == 0) {
1875                 return true;
1876         }
1877         if ((buf = ucl_mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
1878                 ucl_create_err (&parser->err, "cannot mmap fd %d: %s",
1879                         fd, strerror (errno));
1880                 return false;
1881         }
1882
1883         if (parser->cur_file) {
1884                 free (parser->cur_file);
1885         }
1886         parser->cur_file = NULL;
1887         len = st.st_size;
1888         ret = ucl_parser_add_chunk_full (parser, buf, len, priority, strat,
1889                         parse_type);
1890
1891         if (len > 0) {
1892                 ucl_munmap (buf, len);
1893         }
1894
1895         return ret;
1896 }
1897
1898 bool
1899 ucl_parser_add_fd_priority (struct ucl_parser *parser, int fd,
1900                 unsigned priority)
1901 {
1902         if (parser == NULL) {
1903                 return false;
1904         }
1905
1906         return ucl_parser_add_fd_full(parser, fd, parser->default_priority,
1907                         UCL_DUPLICATE_APPEND, UCL_PARSE_UCL);
1908 }
1909
1910 bool
1911 ucl_parser_add_fd (struct ucl_parser *parser, int fd)
1912 {
1913         if (parser == NULL) {
1914                 return false;
1915         }
1916
1917         return ucl_parser_add_fd_priority(parser, fd, parser->default_priority);
1918 }
1919
1920 size_t
1921 ucl_strlcpy (char *dst, const char *src, size_t siz)
1922 {
1923         char *d = dst;
1924         const char *s = src;
1925         size_t n = siz;
1926
1927         /* Copy as many bytes as will fit */
1928         if (n != 0) {
1929                 while (--n != 0) {
1930                         if ((*d++ = *s++) == '\0') {
1931                                 break;
1932                         }
1933                 }
1934         }
1935
1936         if (n == 0 && siz != 0) {
1937                 *d = '\0';
1938         }
1939
1940         return (s - src - 1);    /* count does not include NUL */
1941 }
1942
1943 size_t
1944 ucl_strlcpy_unsafe (char *dst, const char *src, size_t siz)
1945 {
1946         memcpy (dst, src, siz - 1);
1947         dst[siz - 1] = '\0';
1948
1949         return siz - 1;
1950 }
1951
1952 size_t
1953 ucl_strlcpy_tolower (char *dst, const char *src, size_t siz)
1954 {
1955         char *d = dst;
1956         const char *s = src;
1957         size_t n = siz;
1958
1959         /* Copy as many bytes as will fit */
1960         if (n != 0) {
1961                 while (--n != 0) {
1962                         if ((*d++ = tolower (*s++)) == '\0') {
1963                                 break;
1964                         }
1965                 }
1966         }
1967
1968         if (n == 0 && siz != 0) {
1969                 *d = '\0';
1970         }
1971
1972         return (s - src);    /* count does not include NUL */
1973 }
1974
1975 /*
1976  * Find the first occurrence of find in s
1977  */
1978 char *
1979 ucl_strnstr (const char *s, const char *find, int len)
1980 {
1981         char c, sc;
1982         int mlen;
1983
1984         if ((c = *find++) != 0) {
1985                 mlen = strlen (find);
1986                 do {
1987                         do {
1988                                 if ((sc = *s++) == 0 || len-- == 0)
1989                                         return (NULL);
1990                         } while (sc != c);
1991                 } while (strncmp (s, find, mlen) != 0);
1992                 s--;
1993         }
1994         return ((char *)s);
1995 }
1996
1997 /*
1998  * Find the first occurrence of find in s, ignore case.
1999  */
2000 char *
2001 ucl_strncasestr (const char *s, const char *find, int len)
2002 {
2003         char c, sc;
2004         int mlen;
2005
2006         if ((c = *find++) != 0) {
2007                 c = tolower (c);
2008                 mlen = strlen (find);
2009                 do {
2010                         do {
2011                                 if ((sc = *s++) == 0 || len-- == 0)
2012                                         return (NULL);
2013                         } while (tolower (sc) != c);
2014                 } while (strncasecmp (s, find, mlen) != 0);
2015                 s--;
2016         }
2017         return ((char *)s);
2018 }
2019
2020 ucl_object_t *
2021 ucl_object_fromstring_common (const char *str, size_t len, enum ucl_string_flags flags)
2022 {
2023         ucl_object_t *obj;
2024         const char *start, *end, *p, *pos;
2025         char *dst, *d;
2026         size_t escaped_len;
2027
2028         if (str == NULL) {
2029                 return NULL;
2030         }
2031
2032         obj = ucl_object_new ();
2033         if (obj) {
2034                 if (len == 0) {
2035                         len = strlen (str);
2036                 }
2037                 if (flags & UCL_STRING_TRIM) {
2038                         /* Skip leading spaces */
2039                         for (start = str; (size_t)(start - str) < len; start ++) {
2040                                 if (!ucl_test_character (*start, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
2041                                         break;
2042                                 }
2043                         }
2044                         /* Skip trailing spaces */
2045                         for (end = str + len - 1; end > start; end --) {
2046                                 if (!ucl_test_character (*end, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
2047                                         break;
2048                                 }
2049                         }
2050                         end ++;
2051                 }
2052                 else {
2053                         start = str;
2054                         end = str + len;
2055                 }
2056
2057                 obj->type = UCL_STRING;
2058                 if (flags & UCL_STRING_ESCAPE) {
2059                         for (p = start, escaped_len = 0; p < end; p ++, escaped_len ++) {
2060                                 if (ucl_test_character (*p, UCL_CHARACTER_JSON_UNSAFE)) {
2061                                         escaped_len ++;
2062                                 }
2063                         }
2064                         dst = malloc (escaped_len + 1);
2065                         if (dst != NULL) {
2066                                 for (p = start, d = dst; p < end; p ++, d ++) {
2067                                         if (ucl_test_character (*p, UCL_CHARACTER_JSON_UNSAFE)) {
2068                                                 switch (*p) {
2069                                                 case '\n':
2070                                                         *d++ = '\\';
2071                                                         *d = 'n';
2072                                                         break;
2073                                                 case '\r':
2074                                                         *d++ = '\\';
2075                                                         *d = 'r';
2076                                                         break;
2077                                                 case '\b':
2078                                                         *d++ = '\\';
2079                                                         *d = 'b';
2080                                                         break;
2081                                                 case '\t':
2082                                                         *d++ = '\\';
2083                                                         *d = 't';
2084                                                         break;
2085                                                 case '\f':
2086                                                         *d++ = '\\';
2087                                                         *d = 'f';
2088                                                         break;
2089                                                 case '\\':
2090                                                         *d++ = '\\';
2091                                                         *d = '\\';
2092                                                         break;
2093                                                 case '"':
2094                                                         *d++ = '\\';
2095                                                         *d = '"';
2096                                                         break;
2097                                                 }
2098                                         }
2099                                         else {
2100                                                 *d = *p;
2101                                         }
2102                                 }
2103                                 *d = '\0';
2104                                 obj->value.sv = dst;
2105                                 obj->trash_stack[UCL_TRASH_VALUE] = dst;
2106                                 obj->len = escaped_len;
2107                         }
2108                 }
2109                 else {
2110                         dst = malloc (end - start + 1);
2111                         if (dst != NULL) {
2112                                 ucl_strlcpy_unsafe (dst, start, end - start + 1);
2113                                 obj->value.sv = dst;
2114                                 obj->trash_stack[UCL_TRASH_VALUE] = dst;
2115                                 obj->len = end - start;
2116                         }
2117                 }
2118                 if ((flags & UCL_STRING_PARSE) && dst != NULL) {
2119                         /* Parse what we have */
2120                         if (flags & UCL_STRING_PARSE_BOOLEAN) {
2121                                 if (!ucl_maybe_parse_boolean (obj, dst, obj->len) && (flags & UCL_STRING_PARSE_NUMBER)) {
2122                                         ucl_maybe_parse_number (obj, dst, dst + obj->len, &pos,
2123                                                         flags & UCL_STRING_PARSE_DOUBLE,
2124                                                         flags & UCL_STRING_PARSE_BYTES,
2125                                                         flags & UCL_STRING_PARSE_TIME);
2126                                 }
2127                         }
2128                         else {
2129                                 ucl_maybe_parse_number (obj, dst, dst + obj->len, &pos,
2130                                                 flags & UCL_STRING_PARSE_DOUBLE,
2131                                                 flags & UCL_STRING_PARSE_BYTES,
2132                                                 flags & UCL_STRING_PARSE_TIME);
2133                         }
2134                 }
2135         }
2136
2137         return obj;
2138 }
2139
2140 static bool
2141 ucl_object_insert_key_common (ucl_object_t *top, ucl_object_t *elt,
2142                 const char *key, size_t keylen, bool copy_key, bool merge, bool replace)
2143 {
2144         ucl_object_t *found, *tmp;
2145         const ucl_object_t *cur;
2146         ucl_object_iter_t it = NULL;
2147         const char *p;
2148         int ret = true;
2149
2150         if (elt == NULL || key == NULL) {
2151                 return false;
2152         }
2153
2154         if (top == NULL) {
2155                 return false;
2156         }
2157
2158         if (top->type != UCL_OBJECT) {
2159                 /* It is possible to convert NULL type to an object */
2160                 if (top->type == UCL_NULL) {
2161                         top->type = UCL_OBJECT;
2162                 }
2163                 else {
2164                         /* Refuse converting of other object types */
2165                         return false;
2166                 }
2167         }
2168
2169         if (top->value.ov == NULL) {
2170                 top->value.ov = ucl_hash_create (false);
2171         }
2172
2173         if (keylen == 0) {
2174                 keylen = strlen (key);
2175         }
2176
2177         for (p = key; p < key + keylen; p ++) {
2178                 if (ucl_test_character (*p, UCL_CHARACTER_UCL_UNSAFE)) {
2179                         elt->flags |= UCL_OBJECT_NEED_KEY_ESCAPE;
2180                         break;
2181                 }
2182         }
2183
2184         /* workaround for some use cases */
2185         if (elt->trash_stack[UCL_TRASH_KEY] != NULL &&
2186                         key != (const char *)elt->trash_stack[UCL_TRASH_KEY]) {
2187                 /* Remove copied key */
2188                 free (elt->trash_stack[UCL_TRASH_KEY]);
2189                 elt->trash_stack[UCL_TRASH_KEY] = NULL;
2190                 elt->flags &= ~UCL_OBJECT_ALLOCATED_KEY;
2191         }
2192
2193         elt->key = key;
2194         elt->keylen = keylen;
2195
2196         if (copy_key) {
2197                 ucl_copy_key_trash (elt);
2198         }
2199
2200         found = __DECONST (ucl_object_t *, ucl_hash_search_obj (top->value.ov, elt));
2201
2202         if (found == NULL) {
2203                 top->value.ov = ucl_hash_insert_object (top->value.ov, elt, false);
2204                 top->len ++;
2205                 if (replace) {
2206                         ret = false;
2207                 }
2208         }
2209         else {
2210                 if (replace) {
2211                         ucl_hash_replace (top->value.ov, found, elt);
2212                         ucl_object_unref (found);
2213                 }
2214                 else if (merge) {
2215                         if (found->type != UCL_OBJECT && elt->type == UCL_OBJECT) {
2216                                 /* Insert old elt to new one */
2217                                 ucl_object_insert_key_common (elt, found, found->key,
2218                                                 found->keylen, copy_key, false, false);
2219                                 ucl_hash_delete (top->value.ov, found);
2220                                 top->value.ov = ucl_hash_insert_object (top->value.ov, elt, false);
2221                         }
2222                         else if (found->type == UCL_OBJECT && elt->type != UCL_OBJECT) {
2223                                 /* Insert new to old */
2224                                 ucl_object_insert_key_common (found, elt, elt->key,
2225                                                 elt->keylen, copy_key, false, false);
2226                         }
2227                         else if (found->type == UCL_OBJECT && elt->type == UCL_OBJECT) {
2228                                 /* Mix two hashes */
2229                                 while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) {
2230                                         tmp = ucl_object_ref (cur);
2231                                         ucl_object_insert_key_common (found, tmp, cur->key,
2232                                                         cur->keylen, copy_key, false, false);
2233                                 }
2234                                 ucl_object_unref (elt);
2235                         }
2236                         else {
2237                                 /* Just make a list of scalars */
2238                                 DL_APPEND (found, elt);
2239                         }
2240                 }
2241                 else {
2242                         DL_APPEND (found, elt);
2243                 }
2244         }
2245
2246         return ret;
2247 }
2248
2249 bool
2250 ucl_object_delete_keyl (ucl_object_t *top, const char *key, size_t keylen)
2251 {
2252         ucl_object_t *found;
2253
2254         if (top == NULL || key == NULL) {
2255                 return false;
2256         }
2257
2258         found = __DECONST (ucl_object_t *, ucl_object_lookup_len (top, key, keylen));
2259
2260         if (found == NULL) {
2261                 return false;
2262         }
2263
2264         ucl_hash_delete (top->value.ov, found);
2265         ucl_object_unref (found);
2266         top->len --;
2267
2268         return true;
2269 }
2270
2271 bool
2272 ucl_object_delete_key (ucl_object_t *top, const char *key)
2273 {
2274         return ucl_object_delete_keyl (top, key, strlen (key));
2275 }
2276
2277 ucl_object_t*
2278 ucl_object_pop_keyl (ucl_object_t *top, const char *key, size_t keylen)
2279 {
2280         const ucl_object_t *found;
2281
2282         if (top == NULL || key == NULL) {
2283                 return false;
2284         }
2285         found = ucl_object_lookup_len (top, key, keylen);
2286
2287         if (found == NULL) {
2288                 return NULL;
2289         }
2290         ucl_hash_delete (top->value.ov, found);
2291         top->len --;
2292
2293         return __DECONST (ucl_object_t *, found);
2294 }
2295
2296 ucl_object_t*
2297 ucl_object_pop_key (ucl_object_t *top, const char *key)
2298 {
2299         return ucl_object_pop_keyl (top, key, strlen (key));
2300 }
2301
2302 bool
2303 ucl_object_insert_key (ucl_object_t *top, ucl_object_t *elt,
2304                 const char *key, size_t keylen, bool copy_key)
2305 {
2306         return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, false, false);
2307 }
2308
2309 bool
2310 ucl_object_insert_key_merged (ucl_object_t *top, ucl_object_t *elt,
2311                 const char *key, size_t keylen, bool copy_key)
2312 {
2313         return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, true, false);
2314 }
2315
2316 bool
2317 ucl_object_replace_key (ucl_object_t *top, ucl_object_t *elt,
2318                 const char *key, size_t keylen, bool copy_key)
2319 {
2320         return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, false, true);
2321 }
2322
2323 bool
2324 ucl_object_merge (ucl_object_t *top, ucl_object_t *elt, bool copy)
2325 {
2326         ucl_object_t *cur = NULL, *cp = NULL, *found = NULL;
2327         ucl_object_iter_t iter = NULL;
2328
2329         if (top == NULL || top->type != UCL_OBJECT || elt == NULL || elt->type != UCL_OBJECT) {
2330                 return false;
2331         }
2332
2333         /* Mix two hashes */
2334         while ((cur = (ucl_object_t*)ucl_hash_iterate (elt->value.ov, &iter))) {
2335                 if (copy) {
2336                         cp = ucl_object_copy (cur);
2337                 }
2338                 else {
2339                         cp = ucl_object_ref (cur);
2340                 }
2341                 found = __DECONST(ucl_object_t *, ucl_hash_search (top->value.ov, cp->key, cp->keylen));
2342                 if (found == NULL) {
2343                         /* The key does not exist */
2344                         top->value.ov = ucl_hash_insert_object (top->value.ov, cp, false);
2345                         top->len ++;
2346                 }
2347                 else {
2348                         /* The key already exists, replace it */
2349                         ucl_hash_replace (top->value.ov, found, cp);
2350                         ucl_object_unref (found);
2351                 }
2352         }
2353
2354         return true;
2355 }
2356
2357 const ucl_object_t *
2358 ucl_object_lookup_len (const ucl_object_t *obj, const char *key, size_t klen)
2359 {
2360         const ucl_object_t *ret;
2361         ucl_object_t srch;
2362
2363         if (obj == NULL || obj->type != UCL_OBJECT || key == NULL) {
2364                 return NULL;
2365         }
2366
2367         srch.key = key;
2368         srch.keylen = klen;
2369         ret = ucl_hash_search_obj (obj->value.ov, &srch);
2370
2371         return ret;
2372 }
2373
2374 const ucl_object_t *
2375 ucl_object_lookup (const ucl_object_t *obj, const char *key)
2376 {
2377         if (key == NULL) {
2378                 return NULL;
2379         }
2380
2381         return ucl_object_lookup_len (obj, key, strlen (key));
2382 }
2383
2384 const ucl_object_t*
2385 ucl_object_lookup_any (const ucl_object_t *obj,
2386                 const char *key, ...)
2387 {
2388         va_list ap;
2389         const ucl_object_t *ret = NULL;
2390         const char *nk = NULL;
2391
2392         if (obj == NULL || key == NULL) {
2393                 return NULL;
2394         }
2395
2396         ret = ucl_object_lookup_len (obj, key, strlen (key));
2397
2398         if (ret == NULL) {
2399                 va_start (ap, key);
2400
2401                 while (ret == NULL) {
2402                         nk = va_arg (ap, const char *);
2403
2404                         if (nk == NULL) {
2405                                 break;
2406                         }
2407                         else {
2408                                 ret = ucl_object_lookup_len (obj, nk, strlen (nk));
2409                         }
2410                 }
2411
2412                 va_end (ap);
2413         }
2414
2415         return ret;
2416 }
2417
2418 const ucl_object_t*
2419 ucl_object_iterate (const ucl_object_t *obj, ucl_object_iter_t *iter, bool expand_values)
2420 {
2421         const ucl_object_t *elt = NULL;
2422
2423         if (obj == NULL || iter == NULL) {
2424                 return NULL;
2425         }
2426
2427         if (expand_values) {
2428                 switch (obj->type) {
2429                 case UCL_OBJECT:
2430                         return (const ucl_object_t*)ucl_hash_iterate (obj->value.ov, iter);
2431                         break;
2432                 case UCL_ARRAY: {
2433                         unsigned int idx;
2434                         UCL_ARRAY_GET (vec, obj);
2435                         idx = (unsigned int)(uintptr_t)(*iter);
2436
2437                         if (vec != NULL) {
2438                                 while (idx < kv_size (*vec)) {
2439                                         if ((elt = kv_A (*vec, idx)) != NULL) {
2440                                                 idx ++;
2441                                                 break;
2442                                         }
2443                                         idx ++;
2444                                 }
2445                                 *iter = (void *)(uintptr_t)idx;
2446                         }
2447
2448                         return elt;
2449                         break;
2450                 }
2451                 default:
2452                         /* Go to linear iteration */
2453                         break;
2454                 }
2455         }
2456         /* Treat everything as a linear list */
2457         elt = *iter;
2458         if (elt == NULL) {
2459                 elt = obj;
2460         }
2461         else if (elt == obj) {
2462                 return NULL;
2463         }
2464         *iter = __DECONST (void *, elt->next ? elt->next : obj);
2465         return elt;
2466
2467         /* Not reached */
2468         return NULL;
2469 }
2470
2471 const char safe_iter_magic[4] = {'u', 'i', 't', 'e'};
2472 struct ucl_object_safe_iter {
2473         char magic[4]; /* safety check */
2474         const ucl_object_t *impl_it; /* implicit object iteration */
2475         ucl_object_iter_t expl_it; /* explicit iteration */
2476 };
2477
2478 #define UCL_SAFE_ITER(ptr) (struct ucl_object_safe_iter *)(ptr)
2479 #define UCL_SAFE_ITER_CHECK(it) do { \
2480         assert (it != NULL); \
2481         assert (memcmp (it->magic, safe_iter_magic, sizeof (it->magic)) == 0); \
2482  } while (0)
2483
2484 ucl_object_iter_t
2485 ucl_object_iterate_new (const ucl_object_t *obj)
2486 {
2487         struct ucl_object_safe_iter *it;
2488
2489         it = UCL_ALLOC (sizeof (*it));
2490         if (it != NULL) {
2491                 memcpy (it->magic, safe_iter_magic, sizeof (it->magic));
2492                 it->expl_it = NULL;
2493                 it->impl_it = obj;
2494         }
2495
2496         return (ucl_object_iter_t)it;
2497 }
2498
2499
2500 ucl_object_iter_t
2501 ucl_object_iterate_reset (ucl_object_iter_t it, const ucl_object_t *obj)
2502 {
2503         struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
2504
2505         UCL_SAFE_ITER_CHECK (rit);
2506
2507         if (rit->expl_it != NULL) {
2508                 UCL_FREE (sizeof (*rit->expl_it), rit->expl_it);
2509         }
2510
2511         rit->impl_it = obj;
2512         rit->expl_it = NULL;
2513
2514         return it;
2515 }
2516
2517 const ucl_object_t*
2518 ucl_object_iterate_safe (ucl_object_iter_t it, bool expand_values)
2519 {
2520         return ucl_object_iterate_full (it, expand_values ? UCL_ITERATE_BOTH :
2521                         UCL_ITERATE_IMPLICIT);
2522 }
2523
2524 const ucl_object_t*
2525 ucl_object_iterate_full (ucl_object_iter_t it, enum ucl_iterate_type type)
2526 {
2527         struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
2528         const ucl_object_t *ret = NULL;
2529
2530         UCL_SAFE_ITER_CHECK (rit);
2531
2532         if (rit->impl_it == NULL) {
2533                 return NULL;
2534         }
2535
2536         if (rit->impl_it->type == UCL_OBJECT || rit->impl_it->type == UCL_ARRAY) {
2537                 ret = ucl_object_iterate (rit->impl_it, &rit->expl_it, true);
2538
2539                 if (ret == NULL && (type & UCL_ITERATE_IMPLICIT)) {
2540                         /* Need to switch to another implicit object in chain */
2541                         rit->impl_it = rit->impl_it->next;
2542                         rit->expl_it = NULL;
2543
2544                         return ucl_object_iterate_safe (it, type);
2545                 }
2546         }
2547         else {
2548                 /* Just iterate over the implicit array */
2549                 ret = rit->impl_it;
2550                 rit->impl_it = rit->impl_it->next;
2551
2552                 if (type & UCL_ITERATE_EXPLICIT) {
2553                         /* We flatten objects if need to expand values */
2554                         if (ret->type == UCL_OBJECT || ret->type == UCL_ARRAY) {
2555                                 return ucl_object_iterate_safe (it, type);
2556                         }
2557                 }
2558         }
2559
2560         return ret;
2561 }
2562
2563 void
2564 ucl_object_iterate_free (ucl_object_iter_t it)
2565 {
2566         struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
2567
2568         UCL_SAFE_ITER_CHECK (rit);
2569
2570         if (rit->expl_it != NULL) {
2571                 UCL_FREE (sizeof (*rit->expl_it), rit->expl_it);
2572         }
2573
2574         UCL_FREE (sizeof (*rit), it);
2575 }
2576
2577 const ucl_object_t *
2578 ucl_object_lookup_path (const ucl_object_t *top, const char *path_in) {
2579         return ucl_object_lookup_path_char (top, path_in, '.');
2580 }
2581
2582
2583 const ucl_object_t *
2584 ucl_object_lookup_path_char (const ucl_object_t *top, const char *path_in, const char sep) {
2585         const ucl_object_t *o = NULL, *found;
2586         const char *p, *c;
2587         char *err_str;
2588         unsigned index;
2589
2590         if (path_in == NULL || top == NULL) {
2591                 return NULL;
2592         }
2593
2594         found = NULL;
2595         p = path_in;
2596
2597         /* Skip leading dots */
2598         while (*p == sep) {
2599                 p ++;
2600         }
2601
2602         c = p;
2603         while (*p != '\0') {
2604                 p ++;
2605                 if (*p == sep || *p == '\0') {
2606                         if (p > c) {
2607                                 switch (top->type) {
2608                                 case UCL_ARRAY:
2609                                         /* Key should be an int */
2610                                         index = strtoul (c, &err_str, 10);
2611                                         if (err_str != NULL && (*err_str != sep && *err_str != '\0')) {
2612                                                 return NULL;
2613                                         }
2614                                         o = ucl_array_find_index (top, index);
2615                                         break;
2616                                 default:
2617                                         o = ucl_object_lookup_len (top, c, p - c);
2618                                         break;
2619                                 }
2620                                 if (o == NULL) {
2621                                         return NULL;
2622                                 }
2623                                 top = o;
2624                         }
2625                         if (*p != '\0') {
2626                                 c = p + 1;
2627                         }
2628                 }
2629         }
2630         found = o;
2631
2632         return found;
2633 }
2634
2635
2636 ucl_object_t *
2637 ucl_object_new (void)
2638 {
2639         return ucl_object_typed_new (UCL_NULL);
2640 }
2641
2642 ucl_object_t *
2643 ucl_object_typed_new (ucl_type_t type)
2644 {
2645         return ucl_object_new_full (type, 0);
2646 }
2647
2648 ucl_object_t *
2649 ucl_object_new_full (ucl_type_t type, unsigned priority)
2650 {
2651         ucl_object_t *new;
2652
2653         if (type != UCL_USERDATA) {
2654                 new = UCL_ALLOC (sizeof (ucl_object_t));
2655                 if (new != NULL) {
2656                         memset (new, 0, sizeof (ucl_object_t));
2657                         new->ref = 1;
2658                         new->type = (type <= UCL_NULL ? type : UCL_NULL);
2659                         new->next = NULL;
2660                         new->prev = new;
2661                         ucl_object_set_priority (new, priority);
2662
2663                         if (type == UCL_ARRAY) {
2664                                 new->value.av = UCL_ALLOC (sizeof (ucl_array_t));
2665                                 if (new->value.av) {
2666                                         memset (new->value.av, 0, sizeof (ucl_array_t));
2667                                         UCL_ARRAY_GET (vec, new);
2668
2669                                         /* Preallocate some space for arrays */
2670                                         kv_resize (ucl_object_t *, *vec, 8);
2671                                 }
2672                         }
2673                 }
2674         }
2675         else {
2676                 new = ucl_object_new_userdata (NULL, NULL, NULL);
2677                 ucl_object_set_priority (new, priority);
2678         }
2679
2680         return new;
2681 }
2682
2683 ucl_object_t*
2684 ucl_object_new_userdata (ucl_userdata_dtor dtor,
2685                 ucl_userdata_emitter emitter,
2686                 void *ptr)
2687 {
2688         struct ucl_object_userdata *new;
2689         size_t nsize = sizeof (*new);
2690
2691         new = UCL_ALLOC (nsize);
2692         if (new != NULL) {
2693                 memset (new, 0, nsize);
2694                 new->obj.ref = 1;
2695                 new->obj.type = UCL_USERDATA;
2696                 new->obj.next = NULL;
2697                 new->obj.prev = (ucl_object_t *)new;
2698                 new->dtor = dtor;
2699                 new->emitter = emitter;
2700                 new->obj.value.ud = ptr;
2701         }
2702
2703         return (ucl_object_t *)new;
2704 }
2705
2706 ucl_type_t
2707 ucl_object_type (const ucl_object_t *obj)
2708 {
2709         if (obj == NULL) {
2710                 return UCL_NULL;
2711         }
2712
2713         return obj->type;
2714 }
2715
2716 ucl_object_t*
2717 ucl_object_fromstring (const char *str)
2718 {
2719         return ucl_object_fromstring_common (str, 0, UCL_STRING_ESCAPE);
2720 }
2721
2722 ucl_object_t *
2723 ucl_object_fromlstring (const char *str, size_t len)
2724 {
2725         return ucl_object_fromstring_common (str, len, UCL_STRING_ESCAPE);
2726 }
2727
2728 ucl_object_t *
2729 ucl_object_fromint (int64_t iv)
2730 {
2731         ucl_object_t *obj;
2732
2733         obj = ucl_object_new ();
2734         if (obj != NULL) {
2735                 obj->type = UCL_INT;
2736                 obj->value.iv = iv;
2737         }
2738
2739         return obj;
2740 }
2741
2742 ucl_object_t *
2743 ucl_object_fromdouble (double dv)
2744 {
2745         ucl_object_t *obj;
2746
2747         obj = ucl_object_new ();
2748         if (obj != NULL) {
2749                 obj->type = UCL_FLOAT;
2750                 obj->value.dv = dv;
2751         }
2752
2753         return obj;
2754 }
2755
2756 ucl_object_t*
2757 ucl_object_frombool (bool bv)
2758 {
2759         ucl_object_t *obj;
2760
2761         obj = ucl_object_new ();
2762         if (obj != NULL) {
2763                 obj->type = UCL_BOOLEAN;
2764                 obj->value.iv = bv;
2765         }
2766
2767         return obj;
2768 }
2769
2770 bool
2771 ucl_array_append (ucl_object_t *top, ucl_object_t *elt)
2772 {
2773         UCL_ARRAY_GET (vec, top);
2774
2775         if (elt == NULL || top == NULL) {
2776                 return false;
2777         }
2778
2779         if (vec == NULL) {
2780                 vec = UCL_ALLOC (sizeof (*vec));
2781
2782                 if (vec == NULL) {
2783                         return false;
2784                 }
2785
2786                 kv_init (*vec);
2787                 top->value.av = (void *)vec;
2788         }
2789
2790         kv_push (ucl_object_t *, *vec, elt);
2791
2792         top->len ++;
2793
2794         return true;
2795 }
2796
2797 bool
2798 ucl_array_prepend (ucl_object_t *top, ucl_object_t *elt)
2799 {
2800         UCL_ARRAY_GET (vec, top);
2801
2802         if (elt == NULL || top == NULL) {
2803                 return false;
2804         }
2805
2806         if (vec == NULL) {
2807                 vec = UCL_ALLOC (sizeof (*vec));
2808                 kv_init (*vec);
2809                 top->value.av = (void *)vec;
2810                 kv_push (ucl_object_t *, *vec, elt);
2811         }
2812         else {
2813                 /* Slow O(n) algorithm */
2814                 kv_prepend (ucl_object_t *, *vec, elt);
2815         }
2816
2817         top->len ++;
2818
2819         return true;
2820 }
2821
2822 bool
2823 ucl_array_merge (ucl_object_t *top, ucl_object_t *elt, bool copy)
2824 {
2825         unsigned i;
2826         ucl_object_t *cp = NULL;
2827         ucl_object_t **obj;
2828
2829         if (elt == NULL || top == NULL || top->type != UCL_ARRAY || elt->type != UCL_ARRAY) {
2830                 return false;
2831         }
2832
2833         if (copy) {
2834                 cp = ucl_object_copy (elt);
2835         }
2836         else {
2837                 cp = ucl_object_ref (elt);
2838         }
2839
2840         UCL_ARRAY_GET (v1, top);
2841         UCL_ARRAY_GET (v2, cp);
2842
2843         if (v1 && v2) {
2844                 kv_concat (ucl_object_t *, *v1, *v2);
2845
2846                 for (i = v2->n; i < v1->n; i ++) {
2847                         obj = &kv_A (*v1, i);
2848                         if (*obj == NULL) {
2849                                 continue;
2850                         }
2851                         top->len ++;
2852                 }
2853         }
2854
2855         return true;
2856 }
2857
2858 ucl_object_t *
2859 ucl_array_delete (ucl_object_t *top, ucl_object_t *elt)
2860 {
2861         UCL_ARRAY_GET (vec, top);
2862         ucl_object_t *ret = NULL;
2863         unsigned i;
2864
2865         if (vec == NULL) {
2866                 return NULL;
2867         }
2868
2869         for (i = 0; i < vec->n; i ++) {
2870                 if (kv_A (*vec, i) == elt) {
2871                         kv_del (ucl_object_t *, *vec, i);
2872                         ret = elt;
2873                         top->len --;
2874                         break;
2875                 }
2876         }
2877
2878         return ret;
2879 }
2880
2881 const ucl_object_t *
2882 ucl_array_head (const ucl_object_t *top)
2883 {
2884         UCL_ARRAY_GET (vec, top);
2885
2886         if (vec == NULL || top == NULL || top->type != UCL_ARRAY ||
2887                         top->value.av == NULL) {
2888                 return NULL;
2889         }
2890
2891         return (vec->n > 0 ? vec->a[0] : NULL);
2892 }
2893
2894 const ucl_object_t *
2895 ucl_array_tail (const ucl_object_t *top)
2896 {
2897         UCL_ARRAY_GET (vec, top);
2898
2899         if (top == NULL || top->type != UCL_ARRAY || top->value.av == NULL) {
2900                 return NULL;
2901         }
2902
2903         return (vec->n > 0 ? vec->a[vec->n - 1] : NULL);
2904 }
2905
2906 ucl_object_t *
2907 ucl_array_pop_last (ucl_object_t *top)
2908 {
2909         UCL_ARRAY_GET (vec, top);
2910         ucl_object_t **obj, *ret = NULL;
2911
2912         if (vec != NULL && vec->n > 0) {
2913                 obj = &kv_A (*vec, vec->n - 1);
2914                 ret = *obj;
2915                 kv_del (ucl_object_t *, *vec, vec->n - 1);
2916                 top->len --;
2917         }
2918
2919         return ret;
2920 }
2921
2922 ucl_object_t *
2923 ucl_array_pop_first (ucl_object_t *top)
2924 {
2925         UCL_ARRAY_GET (vec, top);
2926         ucl_object_t **obj, *ret = NULL;
2927
2928         if (vec != NULL && vec->n > 0) {
2929                 obj = &kv_A (*vec, 0);
2930                 ret = *obj;
2931                 kv_del (ucl_object_t *, *vec, 0);
2932                 top->len --;
2933         }
2934
2935         return ret;
2936 }
2937
2938 const ucl_object_t *
2939 ucl_array_find_index (const ucl_object_t *top, unsigned int index)
2940 {
2941         UCL_ARRAY_GET (vec, top);
2942
2943         if (vec != NULL && vec->n > 0 && index < vec->n) {
2944                 return kv_A (*vec, index);
2945         }
2946
2947         return NULL;
2948 }
2949
2950 unsigned int
2951 ucl_array_index_of (ucl_object_t *top, ucl_object_t *elt)
2952 {
2953         UCL_ARRAY_GET (vec, top);
2954         unsigned i;
2955
2956         if (vec == NULL) {
2957                 return (unsigned int)(-1);
2958         }
2959
2960         for (i = 0; i < vec->n; i ++) {
2961                 if (kv_A (*vec, i) == elt) {
2962                         return i;
2963                 }
2964         }
2965
2966         return (unsigned int)(-1);
2967 }
2968
2969 ucl_object_t *
2970 ucl_array_replace_index (ucl_object_t *top, ucl_object_t *elt,
2971         unsigned int index)
2972 {
2973         UCL_ARRAY_GET (vec, top);
2974         ucl_object_t *ret = NULL;
2975
2976         if (vec != NULL && vec->n > 0 && index < vec->n) {
2977                 ret = kv_A (*vec, index);
2978                 kv_A (*vec, index) = elt;
2979         }
2980
2981         return ret;
2982 }
2983
2984 ucl_object_t *
2985 ucl_elt_append (ucl_object_t *head, ucl_object_t *elt)
2986 {
2987
2988         if (head == NULL) {
2989                 elt->next = NULL;
2990                 elt->prev = elt;
2991                 head = elt;
2992         }
2993         else {
2994                 elt->prev = head->prev;
2995                 head->prev->next = elt;
2996                 head->prev = elt;
2997                 elt->next = NULL;
2998         }
2999
3000         return head;
3001 }
3002
3003 bool
3004 ucl_object_todouble_safe (const ucl_object_t *obj, double *target)
3005 {
3006         if (obj == NULL || target == NULL) {
3007                 return false;
3008         }
3009         switch (obj->type) {
3010         case UCL_INT:
3011                 *target = obj->value.iv; /* Probaly could cause overflow */
3012                 break;
3013         case UCL_FLOAT:
3014         case UCL_TIME:
3015                 *target = obj->value.dv;
3016                 break;
3017         default:
3018                 return false;
3019         }
3020
3021         return true;
3022 }
3023
3024 double
3025 ucl_object_todouble (const ucl_object_t *obj)
3026 {
3027         double result = 0.;
3028
3029         ucl_object_todouble_safe (obj, &result);
3030         return result;
3031 }
3032
3033 bool
3034 ucl_object_toint_safe (const ucl_object_t *obj, int64_t *target)
3035 {
3036         if (obj == NULL || target == NULL) {
3037                 return false;
3038         }
3039         switch (obj->type) {
3040         case UCL_INT:
3041                 *target = obj->value.iv;
3042                 break;
3043         case UCL_FLOAT:
3044         case UCL_TIME:
3045                 *target = obj->value.dv; /* Loosing of decimal points */
3046                 break;
3047         default:
3048                 return false;
3049         }
3050
3051         return true;
3052 }
3053
3054 int64_t
3055 ucl_object_toint (const ucl_object_t *obj)
3056 {
3057         int64_t result = 0;
3058
3059         ucl_object_toint_safe (obj, &result);
3060         return result;
3061 }
3062
3063 bool
3064 ucl_object_toboolean_safe (const ucl_object_t *obj, bool *target)
3065 {
3066         if (obj == NULL || target == NULL) {
3067                 return false;
3068         }
3069         switch (obj->type) {
3070         case UCL_BOOLEAN:
3071                 *target = (obj->value.iv == true);
3072                 break;
3073         default:
3074                 return false;
3075         }
3076
3077         return true;
3078 }
3079
3080 bool
3081 ucl_object_toboolean (const ucl_object_t *obj)
3082 {
3083         bool result = false;
3084
3085         ucl_object_toboolean_safe (obj, &result);
3086         return result;
3087 }
3088
3089 bool
3090 ucl_object_tostring_safe (const ucl_object_t *obj, const char **target)
3091 {
3092         if (obj == NULL || target == NULL) {
3093                 return false;
3094         }
3095
3096         switch (obj->type) {
3097         case UCL_STRING:
3098                 if (!(obj->flags & UCL_OBJECT_BINARY)) {
3099                         *target = ucl_copy_value_trash (obj);
3100                 }
3101                 break;
3102         default:
3103                 return false;
3104         }
3105
3106         return true;
3107 }
3108
3109 const char *
3110 ucl_object_tostring (const ucl_object_t *obj)
3111 {
3112         const char *result = NULL;
3113
3114         ucl_object_tostring_safe (obj, &result);
3115         return result;
3116 }
3117
3118 const char *
3119 ucl_object_tostring_forced (const ucl_object_t *obj)
3120 {
3121         /* TODO: For binary strings we might encode string here */
3122         if (!(obj->flags & UCL_OBJECT_BINARY)) {
3123                 return ucl_copy_value_trash (obj);
3124         }
3125
3126         return NULL;
3127 }
3128
3129 bool
3130 ucl_object_tolstring_safe (const ucl_object_t *obj, const char **target, size_t *tlen)
3131 {
3132         if (obj == NULL || target == NULL) {
3133                 return false;
3134         }
3135         switch (obj->type) {
3136         case UCL_STRING:
3137                 *target = obj->value.sv;
3138                 if (tlen != NULL) {
3139                         *tlen = obj->len;
3140                 }
3141                 break;
3142         default:
3143                 return false;
3144         }
3145
3146         return true;
3147 }
3148
3149 const char *
3150 ucl_object_tolstring (const ucl_object_t *obj, size_t *tlen)
3151 {
3152         const char *result = NULL;
3153
3154         ucl_object_tolstring_safe (obj, &result, tlen);
3155         return result;
3156 }
3157
3158 const char *
3159 ucl_object_key (const ucl_object_t *obj)
3160 {
3161         return ucl_copy_key_trash (obj);
3162 }
3163
3164 const char *
3165 ucl_object_keyl (const ucl_object_t *obj, size_t *len)
3166 {
3167         if (len == NULL || obj == NULL) {
3168                 return NULL;
3169         }
3170         *len = obj->keylen;
3171         return obj->key;
3172 }
3173
3174 ucl_object_t *
3175 ucl_object_ref (const ucl_object_t *obj)
3176 {
3177         ucl_object_t *res = NULL;
3178
3179         if (obj != NULL) {
3180                 if (obj->flags & UCL_OBJECT_EPHEMERAL) {
3181                         /*
3182                          * Use deep copy for ephemeral objects, note that its refcount
3183                          * is NOT increased, since ephemeral objects does not need refcount
3184                          * at all
3185                          */
3186                         res = ucl_object_copy (obj);
3187                 }
3188                 else {
3189                         res = __DECONST (ucl_object_t *, obj);
3190 #ifdef HAVE_ATOMIC_BUILTINS
3191                         (void)__sync_add_and_fetch (&res->ref, 1);
3192 #else
3193                         res->ref ++;
3194 #endif
3195                 }
3196         }
3197         return res;
3198 }
3199
3200 static ucl_object_t *
3201 ucl_object_copy_internal (const ucl_object_t *other, bool allow_array)
3202 {
3203
3204         ucl_object_t *new;
3205         ucl_object_iter_t it = NULL;
3206         const ucl_object_t *cur;
3207
3208         new = malloc (sizeof (*new));
3209
3210         if (new != NULL) {
3211                 memcpy (new, other, sizeof (*new));
3212                 if (other->flags & UCL_OBJECT_EPHEMERAL) {
3213                         /* Copied object is always non ephemeral */
3214                         new->flags &= ~UCL_OBJECT_EPHEMERAL;
3215                 }
3216                 new->ref = 1;
3217                 /* Unlink from others */
3218                 new->next = NULL;
3219                 new->prev = new;
3220
3221                 /* deep copy of values stored */
3222                 if (other->trash_stack[UCL_TRASH_KEY] != NULL) {
3223                         new->trash_stack[UCL_TRASH_KEY] =
3224                                         strdup (other->trash_stack[UCL_TRASH_KEY]);
3225                         if (other->key == (const char *)other->trash_stack[UCL_TRASH_KEY]) {
3226                                 new->key = new->trash_stack[UCL_TRASH_KEY];
3227                         }
3228                 }
3229                 if (other->trash_stack[UCL_TRASH_VALUE] != NULL) {
3230                         new->trash_stack[UCL_TRASH_VALUE] =
3231                                         strdup (other->trash_stack[UCL_TRASH_VALUE]);
3232                         if (new->type == UCL_STRING) {
3233                                 new->value.sv = new->trash_stack[UCL_TRASH_VALUE];
3234                         }
3235                 }
3236
3237                 if (other->type == UCL_ARRAY || other->type == UCL_OBJECT) {
3238                         /* reset old value */
3239                         memset (&new->value, 0, sizeof (new->value));
3240
3241                         while ((cur = ucl_object_iterate (other, &it, true)) != NULL) {
3242                                 if (other->type == UCL_ARRAY) {
3243                                         ucl_array_append (new, ucl_object_copy_internal (cur, false));
3244                                 }
3245                                 else {
3246                                         ucl_object_t *cp = ucl_object_copy_internal (cur, true);
3247                                         if (cp != NULL) {
3248                                                 ucl_object_insert_key (new, cp, cp->key, cp->keylen,
3249                                                                 false);
3250                                         }
3251                                 }
3252                         }
3253                 }
3254                 else if (allow_array && other->next != NULL) {
3255                         LL_FOREACH (other->next, cur) {
3256                                 ucl_object_t *cp = ucl_object_copy_internal (cur, false);
3257                                 if (cp != NULL) {
3258                                         DL_APPEND (new, cp);
3259                                 }
3260                         }
3261                 }
3262         }
3263
3264         return new;
3265 }
3266
3267 ucl_object_t *
3268 ucl_object_copy (const ucl_object_t *other)
3269 {
3270         return ucl_object_copy_internal (other, true);
3271 }
3272
3273 void
3274 ucl_object_unref (ucl_object_t *obj)
3275 {
3276         if (obj != NULL) {
3277 #ifdef HAVE_ATOMIC_BUILTINS
3278                 unsigned int rc = __sync_sub_and_fetch (&obj->ref, 1);
3279                 if (rc == 0) {
3280 #else
3281                 if (--obj->ref == 0) {
3282 #endif
3283                         ucl_object_free_internal (obj, true, ucl_object_dtor_unref);
3284                 }
3285         }
3286 }
3287
3288 int
3289 ucl_object_compare (const ucl_object_t *o1, const ucl_object_t *o2)
3290 {
3291         const ucl_object_t *it1, *it2;
3292         ucl_object_iter_t iter = NULL;
3293         int ret = 0;
3294
3295         if (o1->type != o2->type) {
3296                 return (o1->type) - (o2->type);
3297         }
3298
3299         switch (o1->type) {
3300         case UCL_STRING:
3301                 if (o1->len == o2->len && o1->len > 0) {
3302                         ret = strcmp (ucl_object_tostring(o1), ucl_object_tostring(o2));
3303                 }
3304                 else {
3305                         ret = o1->len - o2->len;
3306                 }
3307                 break;
3308         case UCL_FLOAT:
3309         case UCL_INT:
3310         case UCL_TIME:
3311                 ret = ucl_object_todouble (o1) - ucl_object_todouble (o2);
3312                 break;
3313         case UCL_BOOLEAN:
3314                 ret = ucl_object_toboolean (o1) - ucl_object_toboolean (o2);
3315                 break;
3316         case UCL_ARRAY:
3317                 if (o1->len == o2->len && o1->len > 0) {
3318                         UCL_ARRAY_GET (vec1, o1);
3319                         UCL_ARRAY_GET (vec2, o2);
3320                         unsigned i;
3321
3322                         /* Compare all elements in both arrays */
3323                         for (i = 0; i < vec1->n; i ++) {
3324                                 it1 = kv_A (*vec1, i);
3325                                 it2 = kv_A (*vec2, i);
3326
3327                                 if (it1 == NULL && it2 != NULL) {
3328                                         return -1;
3329                                 }
3330                                 else if (it2 == NULL && it1 != NULL) {
3331                                         return 1;
3332                                 }
3333                                 else if (it1 != NULL && it2 != NULL) {
3334                                         ret = ucl_object_compare (it1, it2);
3335                                         if (ret != 0) {
3336                                                 break;
3337                                         }
3338                                 }
3339                         }
3340                 }
3341                 else {
3342                         ret = o1->len - o2->len;
3343                 }
3344                 break;
3345         case UCL_OBJECT:
3346                 if (o1->len == o2->len && o1->len > 0) {
3347                         while ((it1 = ucl_object_iterate (o1, &iter, true)) != NULL) {
3348                                 it2 = ucl_object_lookup (o2, ucl_object_key (it1));
3349                                 if (it2 == NULL) {
3350                                         ret = 1;
3351                                         break;
3352                                 }
3353                                 ret = ucl_object_compare (it1, it2);
3354                                 if (ret != 0) {
3355                                         break;
3356                                 }
3357                         }
3358                 }
3359                 else {
3360                         ret = o1->len - o2->len;
3361                 }
3362                 break;
3363         default:
3364                 ret = 0;
3365                 break;
3366         }
3367
3368         return ret;
3369 }
3370
3371 int
3372 ucl_object_compare_qsort (const ucl_object_t **o1,
3373                 const ucl_object_t **o2)
3374 {
3375         return ucl_object_compare (*o1, *o2);
3376 }
3377
3378 void
3379 ucl_object_array_sort (ucl_object_t *ar,
3380                 int (*cmp)(const ucl_object_t **o1, const ucl_object_t **o2))
3381 {
3382         UCL_ARRAY_GET (vec, ar);
3383
3384         if (cmp == NULL || ar == NULL || ar->type != UCL_ARRAY) {
3385                 return;
3386         }
3387
3388         qsort (vec->a, vec->n, sizeof (ucl_object_t *),
3389                         (int (*)(const void *, const void *))cmp);
3390 }
3391
3392 #define PRIOBITS 4
3393
3394 unsigned int
3395 ucl_object_get_priority (const ucl_object_t *obj)
3396 {
3397         if (obj == NULL) {
3398                 return 0;
3399         }
3400
3401         return (obj->flags >> ((sizeof (obj->flags) * NBBY) - PRIOBITS));
3402 }
3403
3404 void
3405 ucl_object_set_priority (ucl_object_t *obj,
3406                 unsigned int priority)
3407 {
3408         if (obj != NULL) {
3409                 priority &= (0x1 << PRIOBITS) - 1;
3410                 priority <<= ((sizeof (obj->flags) * NBBY) - PRIOBITS);
3411                 priority |= obj->flags & ((1 << ((sizeof (obj->flags) * NBBY) -
3412                                 PRIOBITS)) - 1);
3413                 obj->flags = priority;
3414         }
3415 }
3416
3417 bool
3418 ucl_object_string_to_type (const char *input, ucl_type_t *res)
3419 {
3420         if (strcasecmp (input, "object") == 0) {
3421                 *res = UCL_OBJECT;
3422         }
3423         else if (strcasecmp (input, "array") == 0) {
3424                 *res = UCL_ARRAY;
3425         }
3426         else if (strcasecmp (input, "integer") == 0) {
3427                 *res = UCL_INT;
3428         }
3429         else if (strcasecmp (input, "number") == 0) {
3430                 *res = UCL_FLOAT;
3431         }
3432         else if (strcasecmp (input, "string") == 0) {
3433                 *res = UCL_STRING;
3434         }
3435         else if (strcasecmp (input, "boolean") == 0) {
3436                 *res = UCL_BOOLEAN;
3437         }
3438         else if (strcasecmp (input, "null") == 0) {
3439                 *res = UCL_NULL;
3440         }
3441         else if (strcasecmp (input, "userdata") == 0) {
3442                 *res = UCL_USERDATA;
3443         }
3444         else {
3445                 return false;
3446         }
3447
3448         return true;
3449 }
3450
3451 const char *
3452 ucl_object_type_to_string (ucl_type_t type)
3453 {
3454         const char *res = "unknown";
3455
3456         switch (type) {
3457         case UCL_OBJECT:
3458                 res = "object";
3459                 break;
3460         case UCL_ARRAY:
3461                 res = "array";
3462                 break;
3463         case UCL_INT:
3464                 res = "integer";
3465                 break;
3466         case UCL_FLOAT:
3467         case UCL_TIME:
3468                 res = "number";
3469                 break;
3470         case UCL_STRING:
3471                 res = "string";
3472                 break;
3473         case UCL_BOOLEAN:
3474                 res = "boolean";
3475                 break;
3476         case UCL_USERDATA:
3477                 res = "userdata";
3478                 break;
3479         case UCL_NULL:
3480                 res = "null";
3481                 break;
3482         }
3483
3484         return res;
3485 }
3486
3487 const ucl_object_t *
3488 ucl_parser_get_comments (struct ucl_parser *parser)
3489 {
3490         if (parser && parser->comments) {
3491                 return parser->comments;
3492         }
3493
3494         return NULL;
3495 }
3496
3497 const ucl_object_t *
3498 ucl_comments_find (const ucl_object_t *comments,
3499                 const ucl_object_t *srch)
3500 {
3501         if (comments && srch) {
3502                 return ucl_object_lookup_len (comments, (const char *)&srch,
3503                                 sizeof (void *));
3504         }
3505
3506         return NULL;
3507 }
3508
3509 bool
3510 ucl_comments_move (ucl_object_t *comments,
3511                 const ucl_object_t *from, const ucl_object_t *to)
3512 {
3513         const ucl_object_t *found;
3514         ucl_object_t *obj;
3515
3516         if (comments && from && to) {
3517                 found = ucl_object_lookup_len (comments,
3518                                 (const char *)&from, sizeof (void *));
3519
3520                 if (found) {
3521                         /* Replace key */
3522                         obj = ucl_object_ref (found);
3523                         ucl_object_delete_keyl (comments, (const char *)&from,
3524                                         sizeof (void *));
3525                         ucl_object_insert_key (comments, obj, (const char *)&to,
3526                                         sizeof (void *), true);
3527
3528                         return true;
3529                 }
3530         }
3531
3532         return false;
3533 }
3534
3535 void
3536 ucl_comments_add (ucl_object_t *comments, const ucl_object_t *obj,
3537                 const char *comment)
3538 {
3539         if (comments && obj && comment) {
3540                 ucl_object_insert_key (comments, ucl_object_fromstring (comment),
3541                                 (const char *)&obj, sizeof (void *), true);
3542         }
3543 }