contrib/libucl/src/ucl_util.c

   1 /* Copyright (c) 2013, Vsevolod Stakhov
   2  * Copyright (c) 2015 Allan Jude <allanjude@freebsd.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *       * Redistributions of source code must retain the above copyright
   8  *         notice, this list of conditions and the following disclaimer.
   9  *       * Redistributions in binary form must reproduce the above copyright
  10  *         notice, this list of conditions and the following disclaimer in the
  11  *         documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  15  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  16  * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
  17  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  18  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  19  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  20  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  22  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  23  */
  24
  25 #include "ucl.h"
  26 #include "ucl_internal.h"
  27 #include "ucl_chartable.h"
  28 #include "kvec.h"
  29 #include <stdarg.h>
  30 #include <stdio.h> /* for snprintf */
  31
  32 #ifndef _WIN32
  33 #include <glob.h>
  34 #endif
  35
  36 #ifdef HAVE_LIBGEN_H
  37 #include <libgen.h> /* For dirname */
  38 #endif
  39
  40 typedef kvec_t(ucl_object_t *) ucl_array_t;
  41
  42 #define UCL_ARRAY_GET(ar, obj) ucl_array_t *ar = \
  43         (ucl_array_t *)((obj) != NULL ? (obj)->value.av : NULL)
  44
  45 #ifdef HAVE_OPENSSL
  46 #include <openssl/err.h>
  47 #include <openssl/sha.h>
  48 #include <openssl/rsa.h>
  49 #include <openssl/ssl.h>
  50 #include <openssl/evp.h>
  51 #endif
  52
  53 #ifdef CURL_FOUND
  54 /* Seems to be broken */
  55 #define CURL_DISABLE_TYPECHECK 1
  56 #include <curl/curl.h>
  57 #endif
  58 #ifdef HAVE_FETCH_H
  59 #include <fetch.h>
  60 #endif
  61
  62 #ifdef _WIN32
  63 #include <windows.h>
  64
  65 #ifndef PROT_READ
  66 #define PROT_READ       1
  67 #endif
  68 #ifndef PROT_WRITE
  69 #define PROT_WRITE      2
  70 #endif
  71 #ifndef PROT_READWRITE
  72 #define PROT_READWRITE  3
  73 #endif
  74 #ifndef MAP_SHARED
  75 #define MAP_SHARED      1
  76 #endif
  77 #ifndef MAP_PRIVATE
  78 #define MAP_PRIVATE     2
  79 #endif
  80 #ifndef MAP_FAILED
  81 #define MAP_FAILED      ((void *) -1)
  82 #endif
  83
  84 #ifdef _WIN32
  85 #include <limits.h>
  86 #define NBBY CHAR_BIT
  87 #endif
  88
  89 static void *ucl_mmap(char *addr, size_t length, int prot, int access, int fd, off_t offset)
  90 {
  91         void *map = NULL;
  92         HANDLE handle = INVALID_HANDLE_VALUE;
  93
  94         switch (prot) {
  95         default:
  96         case PROT_READ:
  97                 {
  98                         handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READONLY, 0, length, 0);
  99                         if (!handle) break;
 100                         map = (void *) MapViewOfFile(handle, FILE_MAP_READ, 0, 0, length);
 101                         CloseHandle(handle);
 102                         break;
 103                 }
 104         case PROT_WRITE:
 105                 {
 106                         handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READWRITE, 0, length, 0);
 107                         if (!handle) break;
 108                         map = (void *) MapViewOfFile(handle, FILE_MAP_WRITE, 0, 0, length);
 109                         CloseHandle(handle);
 110                         break;
 111                 }
 112         case PROT_READWRITE:
 113                 {
 114                         handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READWRITE, 0, length, 0);
 115                         if (!handle) break;
 116                         map = (void *) MapViewOfFile(handle, FILE_MAP_ALL_ACCESS, 0, 0, length);
 117                         CloseHandle(handle);
 118                         break;
 119                 }
 120         }
 121         if (map == (void *) NULL) {
 122                 return (void *) MAP_FAILED;
 123         }
 124         return (void *) ((char *) map + offset);
 125 }
 126
 127 static int ucl_munmap(void *map,size_t length)
 128 {
 129         if (!UnmapViewOfFile(map)) {
 130                 return(-1);
 131         }
 132         return(0);
 133 }
 134
 135 static char* ucl_realpath(const char *path, char *resolved_path) {
 136     char *p;
 137     char tmp[MAX_PATH + 1];
 138     strncpy(tmp, path, sizeof(tmp)-1);
 139     p = tmp;
 140     while(*p) {
 141         if (*p == '/') *p = '\\';
 142         p++;
 143     }
 144     return _fullpath(resolved_path, tmp, MAX_PATH);
 145 }
 146 #else
 147 #define ucl_mmap mmap
 148 #define ucl_munmap munmap
 149 #define ucl_realpath realpath
 150 #endif
 151
 152 typedef void (*ucl_object_dtor) (ucl_object_t *obj);
 153 static void ucl_object_free_internal (ucl_object_t *obj, bool allow_rec,
 154                 ucl_object_dtor dtor);
 155 static void ucl_object_dtor_unref (ucl_object_t *obj);
 156
 157 static void
 158 ucl_object_dtor_free (ucl_object_t *obj)
 159 {
 160         if (obj->trash_stack[UCL_TRASH_KEY] != NULL) {
 161                 UCL_FREE (obj->hh.keylen, obj->trash_stack[UCL_TRASH_KEY]);
 162         }
 163         if (obj->trash_stack[UCL_TRASH_VALUE] != NULL) {
 164                 UCL_FREE (obj->len, obj->trash_stack[UCL_TRASH_VALUE]);
 165         }
 166         /* Do not free ephemeral objects */
 167         if ((obj->flags & UCL_OBJECT_EPHEMERAL) == 0) {
 168                 if (obj->type != UCL_USERDATA) {
 169                         UCL_FREE (sizeof (ucl_object_t), obj);
 170                 }
 171                 else {
 172                         struct ucl_object_userdata *ud = (struct ucl_object_userdata *)obj;
 173                         if (ud->dtor) {
 174                                 ud->dtor (obj->value.ud);
 175                         }
 176                         UCL_FREE (sizeof (*ud), obj);
 177                 }
 178         }
 179 }
 180
 181 /*
 182  * This is a helper function that performs exactly the same as
 183  * `ucl_object_unref` but it doesn't iterate over elements allowing
 184  * to use it for individual elements of arrays and multiple values
 185  */
 186 static void
 187 ucl_object_dtor_unref_single (ucl_object_t *obj)
 188 {
 189         if (obj != NULL) {
 190 #ifdef HAVE_ATOMIC_BUILTINS
 191                 unsigned int rc = __sync_sub_and_fetch (&obj->ref, 1);
 192                 if (rc == 0) {
 193 #else
 194                 if (--obj->ref == 0) {
 195 #endif
 196                         ucl_object_free_internal (obj, false, ucl_object_dtor_unref);
 197                 }
 198         }
 199 }
 200
 201 static void
 202 ucl_object_dtor_unref (ucl_object_t *obj)
 203 {
 204         if (obj->ref == 0) {
 205                 ucl_object_dtor_free (obj);
 206         }
 207         else {
 208                 /* This may cause dtor unref being called one more time */
 209                 ucl_object_dtor_unref_single (obj);
 210         }
 211 }
 212
 213 static void
 214 ucl_object_free_internal (ucl_object_t *obj, bool allow_rec, ucl_object_dtor dtor)
 215 {
 216         ucl_object_t *tmp, *sub;
 217
 218         while (obj != NULL) {
 219                 if (obj->type == UCL_ARRAY) {
 220                         UCL_ARRAY_GET (vec, obj);
 221                         unsigned int i;
 222
 223                         if (vec != NULL) {
 224                                 for (i = 0; i < vec->n; i ++) {
 225                                         sub = kv_A (*vec, i);
 226                                         if (sub != NULL) {
 227                                                 tmp = sub;
 228                                                 while (sub) {
 229                                                         tmp = sub->next;
 230                                                         dtor (sub);
 231                                                         sub = tmp;
 232                                                 }
 233                                         }
 234                                 }
 235                                 kv_destroy (*vec);
 236                                 UCL_FREE (sizeof (*vec), vec);
 237                         }
 238                         obj->value.av = NULL;
 239                 }
 240                 else if (obj->type == UCL_OBJECT) {
 241                         if (obj->value.ov != NULL) {
 242                                 ucl_hash_destroy (obj->value.ov, (ucl_hash_free_func)dtor);
 243                         }
 244                         obj->value.ov = NULL;
 245                 }
 246                 tmp = obj->next;
 247                 dtor (obj);
 248                 obj = tmp;
 249
 250                 if (!allow_rec) {
 251                         break;
 252                 }
 253         }
 254 }
 255
 256 void
 257 ucl_object_free (ucl_object_t *obj)
 258 {
 259         ucl_object_free_internal (obj, true, ucl_object_dtor_free);
 260 }
 261
 262 size_t
 263 ucl_unescape_json_string (char *str, size_t len)
 264 {
 265         char *t = str, *h = str;
 266         int i, uval;
 267
 268         if (len <= 1) {
 269                 return len;
 270         }
 271         /* t is target (tortoise), h is source (hare) */
 272
 273         while (len) {
 274                 if (*h == '\\') {
 275                         h ++;
 276
 277                         if (len == 1) {
 278                                 /*
 279                                  * If \ is last, then do not try to go further
 280                                  * Issue: #74
 281                                  */
 282                                 len --;
 283                                 *t++ = '\\';
 284                                 continue;
 285                         }
 286
 287                         switch (*h) {
 288                         case 'n':
 289                                 *t++ = '\n';
 290                                 break;
 291                         case 'r':
 292                                 *t++ = '\r';
 293                                 break;
 294                         case 'b':
 295                                 *t++ = '\b';
 296                                 break;
 297                         case 't':
 298                                 *t++ = '\t';
 299                                 break;
 300                         case 'f':
 301                                 *t++ = '\f';
 302                                 break;
 303                         case '\\':
 304                                 *t++ = '\\';
 305                                 break;
 306                         case '"':
 307                                 *t++ = '"';
 308                                 break;
 309                         case 'u':
 310                                 /* Unicode escape */
 311                                 uval = 0;
 312                                 h ++; /* u character */
 313                                 len --;
 314
 315                                 if (len > 3) {
 316                                         for (i = 0; i < 4; i++) {
 317                                                 uval <<= 4;
 318                                                 if (isdigit (h[i])) {
 319                                                         uval += h[i] - '0';
 320                                                 }
 321                                                 else if (h[i] >= 'a' && h[i] <= 'f') {
 322                                                         uval += h[i] - 'a' + 10;
 323                                                 }
 324                                                 else if (h[i] >= 'A' && h[i] <= 'F') {
 325                                                         uval += h[i] - 'A' + 10;
 326                                                 }
 327                                                 else {
 328                                                         break;
 329                                                 }
 330                                         }
 331
 332                                         /* Encode */
 333                                         if(uval < 0x80) {
 334                                                 t[0] = (char)uval;
 335                                                 t ++;
 336                                         }
 337                                         else if(uval < 0x800) {
 338                                                 t[0] = 0xC0 + ((uval & 0x7C0) >> 6);
 339                                                 t[1] = 0x80 + ((uval & 0x03F));
 340                                                 t += 2;
 341                                         }
 342                                         else if(uval < 0x10000) {
 343                                                 t[0] = 0xE0 + ((uval & 0xF000) >> 12);
 344                                                 t[1] = 0x80 + ((uval & 0x0FC0) >> 6);
 345                                                 t[2] = 0x80 + ((uval & 0x003F));
 346                                                 t += 3;
 347                                         }
 348 #if 0
 349                                         /* It's not actually supported now */
 350                                         else if(uval <= 0x10FFFF) {
 351                                                 t[0] = 0xF0 + ((uval & 0x1C0000) >> 18);
 352                                                 t[1] = 0x80 + ((uval & 0x03F000) >> 12);
 353                                                 t[2] = 0x80 + ((uval & 0x000FC0) >> 6);
 354                                                 t[3] = 0x80 + ((uval & 0x00003F));
 355                                                 t += 4;
 356                                         }
 357 #endif
 358                                         else {
 359                                                 *t++ = '?';
 360                                         }
 361
 362                                         /* Consume 4 characters of source */
 363                                         h += 4;
 364                                         len -= 4;
 365
 366                                         if (len > 0) {
 367                                                 len --; /* for '\' character */
 368                                         }
 369                                         continue;
 370                                 }
 371                                 else {
 372                                         *t++ = 'u';
 373                                 }
 374                                 break;
 375                         default:
 376                                 *t++ = *h;
 377                                 break;
 378                         }
 379                         h ++;
 380                         len --;
 381                 }
 382                 else {
 383                         *t++ = *h++;
 384                 }
 385
 386                 if (len > 0) {
 387                         len --;
 388                 }
 389         }
 390         *t = '\0';
 391
 392         return (t - str);
 393 }
 394
 395 char *
 396 ucl_copy_key_trash (const ucl_object_t *obj)
 397 {
 398         ucl_object_t *deconst;
 399
 400         if (obj == NULL) {
 401                 return NULL;
 402         }
 403         if (obj->trash_stack[UCL_TRASH_KEY] == NULL && obj->key != NULL) {
 404                 deconst = __DECONST (ucl_object_t *, obj);
 405                 deconst->trash_stack[UCL_TRASH_KEY] = malloc (obj->keylen + 1);
 406                 if (deconst->trash_stack[UCL_TRASH_KEY] != NULL) {
 407                         memcpy (deconst->trash_stack[UCL_TRASH_KEY], obj->key, obj->keylen);
 408                         deconst->trash_stack[UCL_TRASH_KEY][obj->keylen] = '\0';
 409                 }
 410                 deconst->key = obj->trash_stack[UCL_TRASH_KEY];
 411                 deconst->flags |= UCL_OBJECT_ALLOCATED_KEY;
 412         }
 413
 414         return obj->trash_stack[UCL_TRASH_KEY];
 415 }
 416
 417 char *
 418 ucl_copy_value_trash (const ucl_object_t *obj)
 419 {
 420         ucl_object_t *deconst;
 421
 422         if (obj == NULL) {
 423                 return NULL;
 424         }
 425         if (obj->trash_stack[UCL_TRASH_VALUE] == NULL) {
 426                 deconst = __DECONST (ucl_object_t *, obj);
 427                 if (obj->type == UCL_STRING) {
 428
 429                         /* Special case for strings */
 430                         if (obj->flags & UCL_OBJECT_BINARY) {
 431                                 deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len);
 432                                 if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) {
 433                                         memcpy (deconst->trash_stack[UCL_TRASH_VALUE],
 434                                                         obj->value.sv,
 435                                                         obj->len);
 436                                         deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE];
 437                                 }
 438                         }
 439                         else {
 440                                 deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len + 1);
 441                                 if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) {
 442                                         memcpy (deconst->trash_stack[UCL_TRASH_VALUE],
 443                                                         obj->value.sv,
 444                                                         obj->len);
 445                                         deconst->trash_stack[UCL_TRASH_VALUE][obj->len] = '\0';
 446                                         deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE];
 447                                 }
 448                         }
 449                 }
 450                 else {
 451                         /* Just emit value in json notation */
 452                         deconst->trash_stack[UCL_TRASH_VALUE] = ucl_object_emit_single_json (obj);
 453                         deconst->len = strlen (obj->trash_stack[UCL_TRASH_VALUE]);
 454                 }
 455                 deconst->flags |= UCL_OBJECT_ALLOCATED_VALUE;
 456         }
 457
 458         return obj->trash_stack[UCL_TRASH_VALUE];
 459 }
 460
 461 ucl_object_t*
 462 ucl_parser_get_object (struct ucl_parser *parser)
 463 {
 464         if (parser->state != UCL_STATE_ERROR && parser->top_obj != NULL) {
 465                 return ucl_object_ref (parser->top_obj);
 466         }
 467
 468         return NULL;
 469 }
 470
 471 void
 472 ucl_parser_free (struct ucl_parser *parser)
 473 {
 474         struct ucl_stack *stack, *stmp;
 475         struct ucl_macro *macro, *mtmp;
 476         struct ucl_chunk *chunk, *ctmp;
 477         struct ucl_pubkey *key, *ktmp;
 478         struct ucl_variable *var, *vtmp;
 479         ucl_object_t *tr, *trtmp;
 480
 481         if (parser == NULL) {
 482                 return;
 483         }
 484
 485         if (parser->top_obj != NULL) {
 486                 ucl_object_unref (parser->top_obj);
 487         }
 488
 489         if (parser->includepaths != NULL) {
 490                 ucl_object_unref (parser->includepaths);
 491         }
 492
 493         LL_FOREACH_SAFE (parser->stack, stack, stmp) {
 494                 free (stack);
 495         }
 496         HASH_ITER (hh, parser->macroes, macro, mtmp) {
 497                 free (macro->name);
 498                 HASH_DEL (parser->macroes, macro);
 499                 UCL_FREE (sizeof (struct ucl_macro), macro);
 500         }
 501         LL_FOREACH_SAFE (parser->chunks, chunk, ctmp) {
 502                 UCL_FREE (sizeof (struct ucl_chunk), chunk);
 503         }
 504         LL_FOREACH_SAFE (parser->keys, key, ktmp) {
 505                 UCL_FREE (sizeof (struct ucl_pubkey), key);
 506         }
 507         LL_FOREACH_SAFE (parser->variables, var, vtmp) {
 508                 free (var->value);
 509                 free (var->var);
 510                 UCL_FREE (sizeof (struct ucl_variable), var);
 511         }
 512         LL_FOREACH_SAFE (parser->trash_objs, tr, trtmp) {
 513                 ucl_object_free_internal (tr, false, ucl_object_dtor_free);
 514         }
 515
 516         if (parser->err != NULL) {
 517                 utstring_free (parser->err);
 518         }
 519
 520         if (parser->cur_file) {
 521                 free (parser->cur_file);
 522         }
 523
 524         if (parser->comments) {
 525                 ucl_object_unref (parser->comments);
 526         }
 527
 528         UCL_FREE (sizeof (struct ucl_parser), parser);
 529 }
 530
 531 const char *
 532 ucl_parser_get_error(struct ucl_parser *parser)
 533 {
 534         if (parser == NULL) {
 535                 return NULL;
 536         }
 537
 538         if (parser->err == NULL) {
 539                 return NULL;
 540         }
 541
 542         return utstring_body (parser->err);
 543 }
 544
 545 int
 546 ucl_parser_get_error_code(struct ucl_parser *parser)
 547 {
 548         if (parser == NULL) {
 549                 return 0;
 550         }
 551
 552         return parser->err_code;
 553 }
 554
 555 unsigned
 556 ucl_parser_get_column(struct ucl_parser *parser)
 557 {
 558         if (parser == NULL || parser->chunks == NULL) {
 559                 return 0;
 560         }
 561
 562         return parser->chunks->column;
 563 }
 564
 565 unsigned
 566 ucl_parser_get_linenum(struct ucl_parser *parser)
 567 {
 568         if (parser == NULL || parser->chunks == NULL) {
 569                 return 0;
 570         }
 571
 572         return parser->chunks->line;
 573 }
 574
 575 void
 576 ucl_parser_clear_error(struct ucl_parser *parser)
 577 {
 578         if (parser != NULL && parser->err != NULL) {
 579                 utstring_free(parser->err);
 580                 parser->err = NULL;
 581                 parser->err_code = 0;
 582         }
 583 }
 584
 585 bool
 586 ucl_pubkey_add (struct ucl_parser *parser, const unsigned char *key, size_t len)
 587 {
 588 #ifndef HAVE_OPENSSL
 589         ucl_create_err (&parser->err, "cannot check signatures without openssl");
 590         return false;
 591 #else
 592 # if (OPENSSL_VERSION_NUMBER < 0x10000000L)
 593         ucl_create_err (&parser->err, "cannot check signatures, openssl version is unsupported");
 594         return EXIT_FAILURE;
 595 # else
 596         struct ucl_pubkey *nkey;
 597         BIO *mem;
 598
 599         mem = BIO_new_mem_buf ((void *)key, len);
 600         nkey = UCL_ALLOC (sizeof (struct ucl_pubkey));
 601         if (nkey == NULL) {
 602                 ucl_create_err (&parser->err, "cannot allocate memory for key");
 603                 return false;
 604         }
 605         nkey->key = PEM_read_bio_PUBKEY (mem, &nkey->key, NULL, NULL);
 606         BIO_free (mem);
 607         if (nkey->key == NULL) {
 608                 UCL_FREE (sizeof (struct ucl_pubkey), nkey);
 609                 ucl_create_err (&parser->err, "%s",
 610                                 ERR_error_string (ERR_get_error (), NULL));
 611                 return false;
 612         }
 613         LL_PREPEND (parser->keys, nkey);
 614 # endif
 615 #endif
 616         return true;
 617 }
 618
 619 #ifdef CURL_FOUND
 620 struct ucl_curl_cbdata {
 621         unsigned char *buf;
 622         size_t buflen;
 623 };
 624
 625 static size_t
 626 ucl_curl_write_callback (void* contents, size_t size, size_t nmemb, void* ud)
 627 {
 628         struct ucl_curl_cbdata *cbdata = ud;
 629         size_t realsize = size * nmemb;
 630
 631         cbdata->buf = realloc (cbdata->buf, cbdata->buflen + realsize + 1);
 632         if (cbdata->buf == NULL) {
 633                 return 0;
 634         }
 635
 636         memcpy (&(cbdata->buf[cbdata->buflen]), contents, realsize);
 637         cbdata->buflen += realsize;
 638         cbdata->buf[cbdata->buflen] = 0;
 639
 640         return realsize;
 641 }
 642 #endif
 643
 644 /**
 645  * Fetch a url and save results to the memory buffer
 646  * @param url url to fetch
 647  * @param len length of url
 648  * @param buf target buffer
 649  * @param buflen target length
 650  * @return
 651  */
 652 bool
 653 ucl_fetch_url (const unsigned char *url, unsigned char **buf, size_t *buflen,
 654                 UT_string **err, bool must_exist)
 655 {
 656
 657 #ifdef HAVE_FETCH_H
 658         struct url *fetch_url;
 659         struct url_stat us;
 660         FILE *in;
 661
 662         fetch_url = fetchParseURL (url);
 663         if (fetch_url == NULL) {
 664                 ucl_create_err (err, "invalid URL %s: %s",
 665                                 url, strerror (errno));
 666                 return false;
 667         }
 668         if ((in = fetchXGet (fetch_url, &us, "")) == NULL) {
 669                 if (!must_exist) {
 670                         ucl_create_err (err, "cannot fetch URL %s: %s",
 671                                 url, strerror (errno));
 672                 }
 673                 fetchFreeURL (fetch_url);
 674                 return false;
 675         }
 676
 677         *buflen = us.size;
 678         *buf = malloc (*buflen);
 679         if (*buf == NULL) {
 680                 ucl_create_err (err, "cannot allocate buffer for URL %s: %s",
 681                                 url, strerror (errno));
 682                 fclose (in);
 683                 fetchFreeURL (fetch_url);
 684                 return false;
 685         }
 686
 687         if (fread (*buf, *buflen, 1, in) != 1) {
 688                 ucl_create_err (err, "cannot read URL %s: %s",
 689                                 url, strerror (errno));
 690                 fclose (in);
 691                 fetchFreeURL (fetch_url);
 692                 return false;
 693         }
 694
 695         fetchFreeURL (fetch_url);
 696         return true;
 697 #elif defined(CURL_FOUND)
 698         CURL *curl;
 699         int r;
 700         struct ucl_curl_cbdata cbdata;
 701
 702         curl = curl_easy_init ();
 703         if (curl == NULL) {
 704                 ucl_create_err (err, "CURL interface is broken");
 705                 return false;
 706         }
 707         if ((r = curl_easy_setopt (curl, CURLOPT_URL, url)) != CURLE_OK) {
 708                 ucl_create_err (err, "invalid URL %s: %s",
 709                                 url, curl_easy_strerror (r));
 710                 curl_easy_cleanup (curl);
 711                 return false;
 712         }
 713         curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, ucl_curl_write_callback);
 714         cbdata.buf = NULL;
 715         cbdata.buflen = 0;
 716         curl_easy_setopt (curl, CURLOPT_WRITEDATA, &cbdata);
 717
 718         if ((r = curl_easy_perform (curl)) != CURLE_OK) {
 719                 if (!must_exist) {
 720                         ucl_create_err (err, "error fetching URL %s: %s",
 721                                 url, curl_easy_strerror (r));
 722                 }
 723                 curl_easy_cleanup (curl);
 724                 if (cbdata.buf) {
 725                         free (cbdata.buf);
 726                 }
 727                 return false;
 728         }
 729         *buf = cbdata.buf;
 730         *buflen = cbdata.buflen;
 731
 732         return true;
 733 #else
 734         ucl_create_err (err, "URL support is disabled");
 735         return false;
 736 #endif
 737 }
 738
 739 /**
 740  * Fetch a file and save results to the memory buffer
 741  * @param filename filename to fetch
 742  * @param len length of filename
 743  * @param buf target buffer
 744  * @param buflen target length
 745  * @return
 746  */
 747 bool
 748 ucl_fetch_file (const unsigned char *filename, unsigned char **buf, size_t *buflen,
 749                 UT_string **err, bool must_exist)
 750 {
 751         int fd;
 752         struct stat st;
 753
 754         if (stat (filename, &st) == -1 || !S_ISREG (st.st_mode)) {
 755                 if (must_exist) {
 756                         ucl_create_err (err, "cannot stat file %s: %s",
 757                                         filename, strerror (errno));
 758                 }
 759                 return false;
 760         }
 761         if (st.st_size == 0) {
 762                 /* Do not map empty files */
 763                 *buf = NULL;
 764                 *buflen = 0;
 765         }
 766         else {
 767                 if ((fd = open (filename, O_RDONLY)) == -1) {
 768                         ucl_create_err (err, "cannot open file %s: %s",
 769                                         filename, strerror (errno));
 770                         return false;
 771                 }
 772                 if ((*buf = ucl_mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
 773                         close (fd);
 774                         ucl_create_err (err, "cannot mmap file %s: %s",
 775                                         filename, strerror (errno));
 776                         *buf = NULL;
 777
 778                         return false;
 779                 }
 780                 *buflen = st.st_size;
 781                 close (fd);
 782         }
 783
 784         return true;
 785 }
 786
 787
 788 #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
 789 static inline bool
 790 ucl_sig_check (const unsigned char *data, size_t datalen,
 791                 const unsigned char *sig, size_t siglen, struct ucl_parser *parser)
 792 {
 793         struct ucl_pubkey *key;
 794         char dig[EVP_MAX_MD_SIZE];
 795         unsigned int diglen;
 796         EVP_PKEY_CTX *key_ctx;
 797         EVP_MD_CTX *sign_ctx = NULL;
 798
 799         sign_ctx = EVP_MD_CTX_create ();
 800
 801         LL_FOREACH (parser->keys, key) {
 802                 key_ctx = EVP_PKEY_CTX_new (key->key, NULL);
 803                 if (key_ctx != NULL) {
 804                         if (EVP_PKEY_verify_init (key_ctx) <= 0) {
 805                                 EVP_PKEY_CTX_free (key_ctx);
 806                                 continue;
 807                         }
 808                         if (EVP_PKEY_CTX_set_rsa_padding (key_ctx, RSA_PKCS1_PADDING) <= 0) {
 809                                 EVP_PKEY_CTX_free (key_ctx);
 810                                 continue;
 811                         }
 812                         if (EVP_PKEY_CTX_set_signature_md (key_ctx, EVP_sha256 ()) <= 0) {
 813                                 EVP_PKEY_CTX_free (key_ctx);
 814                                 continue;
 815                         }
 816                         EVP_DigestInit (sign_ctx, EVP_sha256 ());
 817                         EVP_DigestUpdate (sign_ctx, data, datalen);
 818                         EVP_DigestFinal (sign_ctx, dig, &diglen);
 819
 820                         if (EVP_PKEY_verify (key_ctx, sig, siglen, dig, diglen) == 1) {
 821                                 EVP_MD_CTX_destroy (sign_ctx);
 822                                 EVP_PKEY_CTX_free (key_ctx);
 823                                 return true;
 824                         }
 825
 826                         EVP_PKEY_CTX_free (key_ctx);
 827                 }
 828         }
 829
 830         EVP_MD_CTX_destroy (sign_ctx);
 831
 832         return false;
 833 }
 834 #endif
 835
 836 struct ucl_include_params {
 837         bool check_signature;
 838         bool must_exist;
 839         bool use_glob;
 840         bool use_prefix;
 841         bool soft_fail;
 842         bool allow_glob;
 843         unsigned priority;
 844         enum ucl_duplicate_strategy strat;
 845         enum ucl_parse_type parse_type;
 846         const char *prefix;
 847         const char *target;
 848 };
 849
 850 /**
 851  * Include an url to configuration
 852  * @param data
 853  * @param len
 854  * @param parser
 855  * @param err
 856  * @return
 857  */
 858 static bool
 859 ucl_include_url (const unsigned char *data, size_t len,
 860                 struct ucl_parser *parser,
 861                 struct ucl_include_params *params)
 862 {
 863
 864         bool res;
 865         unsigned char *buf = NULL;
 866         size_t buflen = 0;
 867         struct ucl_chunk *chunk;
 868         char urlbuf[PATH_MAX];
 869         int prev_state;
 870
 871         snprintf (urlbuf, sizeof (urlbuf), "%.*s", (int)len, data);
 872
 873         if (!ucl_fetch_url (urlbuf, &buf, &buflen, &parser->err, params->must_exist)) {
 874                 return !params->must_exist;
 875         }
 876
 877         if (params->check_signature) {
 878 #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
 879                 unsigned char *sigbuf = NULL;
 880                 size_t siglen = 0;
 881                 /* We need to check signature first */
 882                 snprintf (urlbuf, sizeof (urlbuf), "%.*s.sig", (int)len, data);
 883                 if (!ucl_fetch_url (urlbuf, &sigbuf, &siglen, &parser->err, true)) {
 884                         return false;
 885                 }
 886                 if (!ucl_sig_check (buf, buflen, sigbuf, siglen, parser)) {
 887                         ucl_create_err (&parser->err, "cannot verify url %s: %s",
 888                                                         urlbuf,
 889                                                         ERR_error_string (ERR_get_error (), NULL));
 890                         if (siglen > 0) {
 891                                 ucl_munmap (sigbuf, siglen);
 892                         }
 893                         return false;
 894                 }
 895                 if (siglen > 0) {
 896                         ucl_munmap (sigbuf, siglen);
 897                 }
 898 #endif
 899         }
 900
 901         prev_state = parser->state;
 902         parser->state = UCL_STATE_INIT;
 903
 904         res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority,
 905                         params->strat, params->parse_type);
 906         if (res == true) {
 907                 /* Remove chunk from the stack */
 908                 chunk = parser->chunks;
 909                 if (chunk != NULL) {
 910                         parser->chunks = chunk->next;
 911                         UCL_FREE (sizeof (struct ucl_chunk), chunk);
 912                 }
 913         }
 914
 915         parser->state = prev_state;
 916         free (buf);
 917
 918         return res;
 919 }
 920
 921 /**
 922  * Include a single file to the parser
 923  * @param data
 924  * @param len
 925  * @param parser
 926  * @param check_signature
 927  * @param must_exist
 928  * @param allow_glob
 929  * @param priority
 930  * @return
 931  */
 932 static bool
 933 ucl_include_file_single (const unsigned char *data, size_t len,
 934                 struct ucl_parser *parser, struct ucl_include_params *params)
 935 {
 936         bool res;
 937         struct ucl_chunk *chunk;
 938         unsigned char *buf = NULL;
 939         char *old_curfile, *ext;
 940         size_t buflen = 0;
 941         char filebuf[PATH_MAX], realbuf[PATH_MAX];
 942         int prev_state;
 943         struct ucl_variable *cur_var, *tmp_var, *old_curdir = NULL,
 944                         *old_filename = NULL;
 945         ucl_object_t *nest_obj = NULL, *old_obj = NULL, *new_obj = NULL;
 946         ucl_hash_t *container = NULL;
 947         struct ucl_stack *st = NULL;
 948
 949         snprintf (filebuf, sizeof (filebuf), "%.*s", (int)len, data);
 950         if (ucl_realpath (filebuf, realbuf) == NULL) {
 951                 if (params->soft_fail) {
 952                         return false;
 953                 }
 954                 if (!params->must_exist) {
 955                         return true;
 956                 }
 957                 ucl_create_err (&parser->err, "cannot open file %s: %s",
 958                                                                         filebuf,
 959                                                                         strerror (errno));
 960                 return false;
 961         }
 962
 963         if (parser->cur_file && strcmp (realbuf, parser->cur_file) == 0) {
 964                 /* We are likely including the file itself */
 965                 if (params->soft_fail) {
 966                         return false;
 967                 }
 968
 969                 ucl_create_err (&parser->err, "trying to include the file %s from itself",
 970                                 realbuf);
 971                 return false;
 972         }
 973
 974         if (!ucl_fetch_file (realbuf, &buf, &buflen, &parser->err, params->must_exist)) {
 975                 if (params->soft_fail) {
 976                         return false;
 977                 }
 978
 979                 return (!params->must_exist || false);
 980         }
 981
 982         if (params->check_signature) {
 983 #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
 984                 unsigned char *sigbuf = NULL;
 985                 size_t siglen = 0;
 986                 /* We need to check signature first */
 987                 snprintf (filebuf, sizeof (filebuf), "%s.sig", realbuf);
 988                 if (!ucl_fetch_file (filebuf, &sigbuf, &siglen, &parser->err, true)) {
 989                         return false;
 990                 }
 991                 if (!ucl_sig_check (buf, buflen, sigbuf, siglen, parser)) {
 992                         ucl_create_err (&parser->err, "cannot verify file %s: %s",
 993                                                         filebuf,
 994                                                         ERR_error_string (ERR_get_error (), NULL));
 995                         if (sigbuf) {
 996                                 ucl_munmap (sigbuf, siglen);
 997                         }
 998                         return false;
 999                 }
1000                 if (sigbuf) {
1001                         ucl_munmap (sigbuf, siglen);
1002                 }
1003 #endif
1004         }
1005
1006         old_curfile = parser->cur_file;
1007         parser->cur_file = strdup (realbuf);
1008
1009         /* Store old file vars */
1010         DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) {
1011                 if (strcmp (cur_var->var, "CURDIR") == 0) {
1012                         old_curdir = cur_var;
1013                         DL_DELETE (parser->variables, cur_var);
1014                 }
1015                 else if (strcmp (cur_var->var, "FILENAME") == 0) {
1016                         old_filename = cur_var;
1017                         DL_DELETE (parser->variables, cur_var);
1018                 }
1019         }
1020
1021         ucl_parser_set_filevars (parser, realbuf, false);
1022
1023         prev_state = parser->state;
1024         parser->state = UCL_STATE_INIT;
1025
1026         if (params->use_prefix && params->prefix == NULL) {
1027                 /* Auto generate a key name based on the included filename */
1028                 params->prefix = basename (realbuf);
1029                 ext = strrchr (params->prefix, '.');
1030                 if (ext != NULL && (strcmp (ext, ".conf") == 0 || strcmp (ext, ".ucl") == 0)) {
1031                         /* Strip off .conf or .ucl */
1032                         *ext = '\0';
1033                 }
1034         }
1035         if (params->prefix != NULL) {
1036                 /* This is a prefixed include */
1037                 container = parser->stack->obj->value.ov;
1038
1039                 old_obj = __DECONST (ucl_object_t *, ucl_hash_search (container,
1040                                 params->prefix, strlen (params->prefix)));
1041
1042                 if (strcasecmp (params->target, "array") == 0 && old_obj == NULL) {
1043                         /* Create an array with key: prefix */
1044                         old_obj = ucl_object_new_full (UCL_ARRAY, params->priority);
1045                         old_obj->key = params->prefix;
1046                         old_obj->keylen = strlen (params->prefix);
1047                         ucl_copy_key_trash(old_obj);
1048                         old_obj->prev = old_obj;
1049                         old_obj->next = NULL;
1050
1051                         container = ucl_hash_insert_object (container, old_obj,
1052                                         parser->flags & UCL_PARSER_KEY_LOWERCASE);
1053                         parser->stack->obj->len ++;
1054
1055                         nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
1056                         nest_obj->prev = nest_obj;
1057                         nest_obj->next = NULL;
1058
1059                         ucl_array_append (old_obj, nest_obj);
1060                 }
1061                 else if (old_obj == NULL) {
1062                         /* Create an object with key: prefix */
1063                         nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
1064
1065                         if (nest_obj == NULL) {
1066                                 ucl_create_err (&parser->err, "cannot allocate memory for an object");
1067                                 if (buf) {
1068                                         ucl_munmap (buf, buflen);
1069                                 }
1070
1071                                 return false;
1072                         }
1073
1074                         nest_obj->key = params->prefix;
1075                         nest_obj->keylen = strlen (params->prefix);
1076                         ucl_copy_key_trash(nest_obj);
1077                         nest_obj->prev = nest_obj;
1078                         nest_obj->next = NULL;
1079
1080                         container = ucl_hash_insert_object (container, nest_obj,
1081                                         parser->flags & UCL_PARSER_KEY_LOWERCASE);
1082                         parser->stack->obj->len ++;
1083                 }
1084                 else if (strcasecmp (params->target, "array") == 0 ||
1085                                 ucl_object_type(old_obj) == UCL_ARRAY) {
1086                         if (ucl_object_type(old_obj) == UCL_ARRAY) {
1087                                 /* Append to the existing array */
1088                                 nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
1089                                 if (nest_obj == NULL) {
1090                                         ucl_create_err (&parser->err, "cannot allocate memory for an object");
1091                                         if (buf) {
1092                                                 ucl_munmap (buf, buflen);
1093                                         }
1094
1095                                         return false;
1096                                 }
1097                                 nest_obj->prev = nest_obj;
1098                                 nest_obj->next = NULL;
1099
1100                                 ucl_array_append (old_obj, nest_obj);
1101                         }
1102                         else {
1103                                 /* Convert the object to an array */
1104                                 new_obj = ucl_object_typed_new (UCL_ARRAY);
1105                                 if (new_obj == NULL) {
1106                                         ucl_create_err (&parser->err, "cannot allocate memory for an object");
1107                                         if (buf) {
1108                                                 ucl_munmap (buf, buflen);
1109                                         }
1110
1111                                         return false;
1112                                 }
1113                                 new_obj->key = old_obj->key;
1114                                 new_obj->keylen = old_obj->keylen;
1115                                 new_obj->flags |= UCL_OBJECT_MULTIVALUE;
1116                                 new_obj->prev = new_obj;
1117                                 new_obj->next = NULL;
1118
1119                                 nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
1120                                 if (nest_obj == NULL) {
1121                                         ucl_create_err (&parser->err, "cannot allocate memory for an object");
1122                                         if (buf) {
1123                                                 ucl_munmap (buf, buflen);
1124                                         }
1125
1126                                         return false;
1127                                 }
1128                                 nest_obj->prev = nest_obj;
1129                                 nest_obj->next = NULL;
1130
1131                                 ucl_array_append (new_obj, old_obj);
1132                                 ucl_array_append (new_obj, nest_obj);
1133                                 ucl_hash_replace (container, old_obj, new_obj);
1134                         }
1135                 }
1136                 else {
1137                         if (ucl_object_type (old_obj) == UCL_OBJECT) {
1138                                 /* Append to existing Object*/
1139                                 nest_obj = old_obj;
1140                         }
1141                         else {
1142                                 /* The key is not an object */
1143                                 ucl_create_err (&parser->err,
1144                                                 "Conflicting type for key: %s",
1145                                                 params->prefix);
1146                                 if (buf) {
1147                                         ucl_munmap (buf, buflen);
1148                                 }
1149
1150                                 return false;
1151                         }
1152                 }
1153
1154                  /* Put all of the content of the include inside that object */
1155                 parser->stack->obj->value.ov = container;
1156
1157                 st = UCL_ALLOC (sizeof (struct ucl_stack));
1158                 if (st == NULL) {
1159                         ucl_create_err (&parser->err, "cannot allocate memory for an object");
1160                         ucl_object_unref (nest_obj);
1161
1162                         if (buf) {
1163                                 ucl_munmap (buf, buflen);
1164                         }
1165
1166                         return false;
1167                 }
1168                 st->obj = nest_obj;
1169                 st->level = parser->stack->level;
1170                 LL_PREPEND (parser->stack, st);
1171                 parser->cur_obj = nest_obj;
1172         }
1173
1174         res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority,
1175                         params->strat, params->parse_type);
1176
1177         if (!res) {
1178                 if (!params->must_exist) {
1179                         /* Free error */
1180                         utstring_free (parser->err);
1181                         parser->err = NULL;
1182                         res = true;
1183                 }
1184         }
1185
1186         /* Stop nesting the include, take 1 level off the stack */
1187         if (params->prefix != NULL && nest_obj != NULL) {
1188                 parser->stack = st->next;
1189                 UCL_FREE (sizeof (struct ucl_stack), st);
1190         }
1191
1192         /* Remove chunk from the stack */
1193         chunk = parser->chunks;
1194         if (chunk != NULL) {
1195                 parser->chunks = chunk->next;
1196                 UCL_FREE (sizeof (struct ucl_chunk), chunk);
1197                 parser->recursion --;
1198         }
1199
1200         /* Restore old file vars */
1201         if (parser->cur_file) {
1202                 free (parser->cur_file);
1203         }
1204
1205         parser->cur_file = old_curfile;
1206         DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) {
1207                 if (strcmp (cur_var->var, "CURDIR") == 0 && old_curdir) {
1208                         DL_DELETE (parser->variables, cur_var);
1209                         free (cur_var->var);
1210                         free (cur_var->value);
1211                         UCL_FREE (sizeof (struct ucl_variable), cur_var);
1212                 }
1213                 else if (strcmp (cur_var->var, "FILENAME") == 0 && old_filename) {
1214                         DL_DELETE (parser->variables, cur_var);
1215                         free (cur_var->var);
1216                         free (cur_var->value);
1217                         UCL_FREE (sizeof (struct ucl_variable), cur_var);
1218                 }
1219         }
1220         if (old_filename) {
1221                 DL_APPEND (parser->variables, old_filename);
1222         }
1223         if (old_curdir) {
1224                 DL_APPEND (parser->variables, old_curdir);
1225         }
1226
1227         parser->state = prev_state;
1228
1229         if (buflen > 0) {
1230                 ucl_munmap (buf, buflen);
1231         }
1232
1233         return res;
1234 }
1235
1236 /**
1237  * Include a file to configuration
1238  * @param data
1239  * @param len
1240  * @param parser
1241  * @param err
1242  * @return
1243  */
1244 static bool
1245 ucl_include_file (const unsigned char *data, size_t len,
1246                 struct ucl_parser *parser, struct ucl_include_params *params)
1247 {
1248         const unsigned char *p = data, *end = data + len;
1249         bool need_glob = false;
1250         int cnt = 0;
1251         char glob_pattern[PATH_MAX];
1252         size_t i;
1253
1254 #ifndef _WIN32
1255         if (!params->allow_glob) {
1256                 return ucl_include_file_single (data, len, parser, params);
1257         }
1258         else {
1259                 /* Check for special symbols in a filename */
1260                 while (p != end) {
1261                         if (*p == '*' || *p == '?') {
1262                                 need_glob = true;
1263                                 break;
1264                         }
1265                         p ++;
1266                 }
1267                 if (need_glob) {
1268                         glob_t globbuf;
1269                         memset (&globbuf, 0, sizeof (globbuf));
1270                         ucl_strlcpy (glob_pattern, (const char *)data,
1271                                 (len + 1 < sizeof (glob_pattern) ? len + 1 : sizeof (glob_pattern)));
1272                         if (glob (glob_pattern, 0, NULL, &globbuf) != 0) {
1273                                 return (!params->must_exist || false);
1274                         }
1275                         for (i = 0; i < globbuf.gl_pathc; i ++) {
1276                                 if (!ucl_include_file_single ((unsigned char *)globbuf.gl_pathv[i],
1277                                                 strlen (globbuf.gl_pathv[i]), parser, params)) {
1278                                         if (params->soft_fail) {
1279                                                 continue;
1280                                         }
1281                                         globfree (&globbuf);
1282                                         return false;
1283                                 }
1284                                 cnt ++;
1285                         }
1286                         globfree (&globbuf);
1287
1288                         if (cnt == 0 && params->must_exist) {
1289                                 ucl_create_err (&parser->err, "cannot match any files for pattern %s",
1290                                         glob_pattern);
1291                                 return false;
1292                         }
1293                 }
1294                 else {
1295                         return ucl_include_file_single (data, len, parser, params);
1296                 }
1297         }
1298 #else
1299         /* Win32 compilers do not support globbing. Therefore, for Win32,
1300            treat allow_glob/need_glob as a NOOP and just return */
1301         return ucl_include_file_single (data, len, parser, params);
1302 #endif
1303
1304         return true;
1305 }
1306
1307 /**
1308  * Common function to handle .*include* macros
1309  * @param data
1310  * @param len
1311  * @param args
1312  * @param parser
1313  * @param default_try
1314  * @param default_sign
1315  * @return
1316  */
1317 static bool
1318 ucl_include_common (const unsigned char *data, size_t len,
1319                 const ucl_object_t *args, struct ucl_parser *parser,
1320                 bool default_try,
1321                 bool default_sign)
1322 {
1323         bool allow_url = false, search = false;
1324         const char *duplicate;
1325         const ucl_object_t *param;
1326         ucl_object_iter_t it = NULL, ip = NULL;
1327         char ipath[PATH_MAX];
1328         struct ucl_include_params params;
1329
1330         /* Default values */
1331         params.soft_fail = default_try;
1332         params.allow_glob = false;
1333         params.check_signature = default_sign;
1334         params.use_prefix = false;
1335         params.target = "object";
1336         params.prefix = NULL;
1337         params.priority = 0;
1338         params.parse_type = UCL_PARSE_UCL;
1339         params.strat = UCL_DUPLICATE_APPEND;
1340         params.must_exist = !default_try;
1341
1342         /* Process arguments */
1343         if (args != NULL && args->type == UCL_OBJECT) {
1344                 while ((param = ucl_object_iterate (args, &it, true)) != NULL) {
1345                         if (param->type == UCL_BOOLEAN) {
1346                                 if (strncmp (param->key, "try", param->keylen) == 0) {
1347                                         params.must_exist = !ucl_object_toboolean (param);
1348                                 }
1349                                 else if (strncmp (param->key, "sign", param->keylen) == 0) {
1350                                         params.check_signature = ucl_object_toboolean (param);
1351                                 }
1352                                 else if (strncmp (param->key, "glob", param->keylen) == 0) {
1353                                         params.allow_glob = ucl_object_toboolean (param);
1354                                 }
1355                                 else if (strncmp (param->key, "url", param->keylen) == 0) {
1356                                         allow_url = ucl_object_toboolean (param);
1357                                 }
1358                                 else if (strncmp (param->key, "prefix", param->keylen) == 0) {
1359                                         params.use_prefix = ucl_object_toboolean (param);
1360                                 }
1361                         }
1362                         else if (param->type == UCL_STRING) {
1363                                 if (strncmp (param->key, "key", param->keylen) == 0) {
1364                                         params.prefix = ucl_object_tostring (param);
1365                                 }
1366                                 else if (strncmp (param->key, "target", param->keylen) == 0) {
1367                                         params.target = ucl_object_tostring (param);
1368                                 }
1369                                 else if (strncmp (param->key, "duplicate", param->keylen) == 0) {
1370                                         duplicate = ucl_object_tostring (param);
1371
1372                                         if (strcmp (duplicate, "append") == 0) {
1373                                                 params.strat = UCL_DUPLICATE_APPEND;
1374                                         }
1375                                         else if (strcmp (duplicate, "merge") == 0) {
1376                                                 params.strat = UCL_DUPLICATE_MERGE;
1377                                         }
1378                                         else if (strcmp (duplicate, "rewrite") == 0) {
1379                                                 params.strat = UCL_DUPLICATE_REWRITE;
1380                                         }
1381                                         else if (strcmp (duplicate, "error") == 0) {
1382                                                 params.strat = UCL_DUPLICATE_ERROR;
1383                                         }
1384                                 }
1385                         }
1386                         else if (param->type == UCL_ARRAY) {
1387                                 if (strncmp (param->key, "path", param->keylen) == 0) {
1388                                         ucl_set_include_path (parser, __DECONST(ucl_object_t *, param));
1389                                 }
1390                         }
1391                         else if (param->type == UCL_INT) {
1392                                 if (strncmp (param->key, "priority", param->keylen) == 0) {
1393                                         params.priority = ucl_object_toint (param);
1394                                 }
1395                         }
1396                 }
1397         }
1398
1399         if (parser->includepaths == NULL) {
1400                 if (allow_url && ucl_strnstr (data, "://", len) != NULL) {
1401                         /* Globbing is not used for URL's */
1402                         return ucl_include_url (data, len, parser, &params);
1403                 }
1404                 else if (data != NULL) {
1405                         /* Try to load a file */
1406                         return ucl_include_file (data, len, parser, &params);
1407                 }
1408         }
1409         else {
1410                 if (allow_url && ucl_strnstr (data, "://", len) != NULL) {
1411                         /* Globbing is not used for URL's */
1412                         return ucl_include_url (data, len, parser, &params);
1413                 }
1414
1415                 ip = ucl_object_iterate_new (parser->includepaths);
1416                 while ((param = ucl_object_iterate_safe (ip, true)) != NULL) {
1417                         if (ucl_object_type(param) == UCL_STRING) {
1418                                 snprintf (ipath, sizeof (ipath), "%s/%.*s", ucl_object_tostring(param),
1419                                                 (int)len, data);
1420                                 if ((search = ucl_include_file (ipath, strlen (ipath),
1421                                                 parser, &params))) {
1422                                         if (!params.allow_glob) {
1423                                                 break;
1424                                         }
1425                                 }
1426                         }
1427                 }
1428                 ucl_object_iterate_free (ip);
1429                 if (search == true) {
1430                         return true;
1431                 }
1432                 else {
1433                         ucl_create_err (&parser->err,
1434                                         "cannot find file: %.*s in search path",
1435                                         (int)len, data);
1436                         return false;
1437                 }
1438         }
1439
1440         return false;
1441 }
1442
1443 /**
1444  * Handle include macro
1445  * @param data include data
1446  * @param len length of data
1447  * @param args UCL object representing arguments to the macro
1448  * @param ud user data
1449  * @return
1450  */
1451 bool
1452 ucl_include_handler (const unsigned char *data, size_t len,
1453                 const ucl_object_t *args, void* ud)
1454 {
1455         struct ucl_parser *parser = ud;
1456
1457         return ucl_include_common (data, len, args, parser, false, false);
1458 }
1459
1460 /**
1461  * Handle includes macro
1462  * @param data include data
1463  * @param len length of data
1464  * @param args UCL object representing arguments to the macro
1465  * @param ud user data
1466  * @return
1467  */
1468 bool
1469 ucl_includes_handler (const unsigned char *data, size_t len,
1470                 const ucl_object_t *args, void* ud)
1471 {
1472         struct ucl_parser *parser = ud;
1473
1474         return ucl_include_common (data, len, args, parser, false, true);
1475 }
1476
1477 /**
1478  * Handle tryinclude macro
1479  * @param data include data
1480  * @param len length of data
1481  * @param args UCL object representing arguments to the macro
1482  * @param ud user data
1483  * @return
1484  */
1485 bool
1486 ucl_try_include_handler (const unsigned char *data, size_t len,
1487                 const ucl_object_t *args, void* ud)
1488 {
1489         struct ucl_parser *parser = ud;
1490
1491         return ucl_include_common (data, len, args, parser, true, false);
1492 }
1493
1494 /**
1495  * Handle priority macro
1496  * @param data include data
1497  * @param len length of data
1498  * @param args UCL object representing arguments to the macro
1499  * @param ud user data
1500  * @return
1501  */
1502 bool
1503 ucl_priority_handler (const unsigned char *data, size_t len,
1504                 const ucl_object_t *args, void* ud)
1505 {
1506         struct ucl_parser *parser = ud;
1507         unsigned priority = 255;
1508         const ucl_object_t *param;
1509         bool found = false;
1510         char *value = NULL, *leftover = NULL;
1511         ucl_object_iter_t it = NULL;
1512
1513         if (parser == NULL) {
1514                 return false;
1515         }
1516
1517         /* Process arguments */
1518         if (args != NULL && args->type == UCL_OBJECT) {
1519                 while ((param = ucl_object_iterate (args, &it, true)) != NULL) {
1520                         if (param->type == UCL_INT) {
1521                                 if (strncmp (param->key, "priority", param->keylen) == 0) {
1522                                         priority = ucl_object_toint (param);
1523                                         found = true;
1524                                 }
1525                         }
1526                 }
1527         }
1528
1529         if (len > 0) {
1530                 value = malloc(len + 1);
1531                 ucl_strlcpy(value, (const char *)data, len + 1);
1532                 priority = strtol(value, &leftover, 10);
1533                 if (*leftover != '\0') {
1534                         ucl_create_err (&parser->err, "Invalid priority value in macro: %s",
1535                                 value);
1536                         free(value);
1537                         return false;
1538                 }
1539                 free(value);
1540                 found = true;
1541         }
1542
1543         if (found == true) {
1544                 parser->chunks->priority = priority;
1545                 return true;
1546         }
1547
1548         ucl_create_err (&parser->err, "Unable to parse priority macro");
1549         return false;
1550 }
1551
1552 /**
1553  * Handle load macro
1554  * @param data include data
1555  * @param len length of data
1556  * @param args UCL object representing arguments to the macro
1557  * @param ud user data
1558  * @return
1559  */
1560 bool
1561 ucl_load_handler (const unsigned char *data, size_t len,
1562                 const ucl_object_t *args, void* ud)
1563 {
1564         struct ucl_parser *parser = ud;
1565         const ucl_object_t *param;
1566         ucl_object_t *obj, *old_obj;
1567         ucl_object_iter_t it = NULL;
1568         bool try_load, multiline, test;
1569         const char *target, *prefix;
1570         char *load_file, *tmp;
1571         unsigned char *buf;
1572         size_t buflen;
1573         unsigned priority;
1574         int64_t iv;
1575         ucl_object_t *container = NULL;
1576         enum ucl_string_flags flags;
1577
1578         /* Default values */
1579         try_load = false;
1580         multiline = false;
1581         test = false;
1582         target = "string";
1583         prefix = NULL;
1584         load_file = NULL;
1585         buf = NULL;
1586         buflen = 0;
1587         priority = 0;
1588         obj = NULL;
1589         old_obj = NULL;
1590         flags = 0;
1591
1592         if (parser == NULL) {
1593                 return false;
1594         }
1595
1596         /* Process arguments */
1597         if (args != NULL && args->type == UCL_OBJECT) {
1598                 while ((param = ucl_object_iterate (args, &it, true)) != NULL) {
1599                         if (param->type == UCL_BOOLEAN) {
1600                                 if (strncmp (param->key, "try", param->keylen) == 0) {
1601                                         try_load = ucl_object_toboolean (param);
1602                                 }
1603                                 else if (strncmp (param->key, "multiline", param->keylen) == 0) {
1604                                         multiline = ucl_object_toboolean (param);
1605                                 }
1606                                 else if (strncmp (param->key, "escape", param->keylen) == 0) {
1607                                         test = ucl_object_toboolean (param);
1608                                         if (test) {
1609                                                 flags |= UCL_STRING_ESCAPE;
1610                                         }
1611                                 }
1612                                 else if (strncmp (param->key, "trim", param->keylen) == 0) {
1613                                         test = ucl_object_toboolean (param);
1614                                         if (test) {
1615                                                 flags |= UCL_STRING_TRIM;
1616                                         }
1617                                 }
1618                         }
1619                         else if (param->type == UCL_STRING) {
1620                                 if (strncmp (param->key, "key", param->keylen) == 0) {
1621                                         prefix = ucl_object_tostring (param);
1622                                 }
1623                                 else if (strncmp (param->key, "target", param->keylen) == 0) {
1624                                         target = ucl_object_tostring (param);
1625                                 }
1626                         }
1627                         else if (param->type == UCL_INT) {
1628                                 if (strncmp (param->key, "priority", param->keylen) == 0) {
1629                                         priority = ucl_object_toint (param);
1630                                 }
1631                         }
1632                 }
1633         }
1634
1635         if (prefix == NULL || strlen (prefix) == 0) {
1636                 ucl_create_err (&parser->err, "No Key specified in load macro");
1637                 return false;
1638         }
1639
1640         if (len > 0) {
1641                 load_file = malloc (len + 1);
1642                 if (!load_file) {
1643                         ucl_create_err (&parser->err, "cannot allocate memory for suffix");
1644
1645                         return false;
1646                 }
1647
1648                 snprintf (load_file, len + 1, "%.*s", (int)len, data);
1649
1650                 if (!ucl_fetch_file (load_file, &buf, &buflen, &parser->err,
1651                                 !try_load)) {
1652                         free (load_file);
1653
1654                         return (try_load || false);
1655                 }
1656
1657                 free (load_file);
1658                 container = parser->stack->obj;
1659                 old_obj = __DECONST (ucl_object_t *, ucl_object_lookup (container,
1660                                 prefix));
1661
1662                 if (old_obj != NULL) {
1663                         ucl_create_err (&parser->err, "Key %s already exists", prefix);
1664                         if (buf) {
1665                                 ucl_munmap (buf, buflen);
1666                         }
1667
1668                         return false;
1669                 }
1670
1671                 if (strcasecmp (target, "string") == 0) {
1672                         obj = ucl_object_fromstring_common (buf, buflen, flags);
1673                         ucl_copy_value_trash (obj);
1674                         if (multiline) {
1675                                 obj->flags |= UCL_OBJECT_MULTILINE;
1676                         }
1677                 }
1678                 else if (strcasecmp (target, "int") == 0) {
1679                         tmp = malloc (buflen + 1);
1680
1681                         if (tmp == NULL) {
1682                                 ucl_create_err (&parser->err, "Memory allocation failed");
1683                                 if (buf) {
1684                                         ucl_munmap (buf, buflen);
1685                                 }
1686
1687                                 return false;
1688                         }
1689
1690                         snprintf (tmp, buflen + 1, "%.*s", (int)buflen, buf);
1691                         iv = strtoll (tmp, NULL, 10);
1692                         obj = ucl_object_fromint (iv);
1693                         free (tmp);
1694                 }
1695
1696                 if (buf) {
1697                         ucl_munmap (buf, buflen);
1698                 }
1699
1700                 if (obj != NULL) {
1701                         obj->key = prefix;
1702                         obj->keylen = strlen (prefix);
1703                         ucl_copy_key_trash (obj);
1704                         obj->prev = obj;
1705                         obj->next = NULL;
1706                         ucl_object_set_priority (obj, priority);
1707                         ucl_object_insert_key (container, obj, obj->key, obj->keylen, false);
1708                 }
1709
1710                 return true;
1711         }
1712
1713         ucl_create_err (&parser->err, "Unable to parse load macro");
1714         return false;
1715 }
1716
1717 bool
1718 ucl_inherit_handler (const unsigned char *data, size_t len,
1719                 const ucl_object_t *args, const ucl_object_t *ctx, void* ud)
1720 {
1721         const ucl_object_t *parent, *cur;
1722         ucl_object_t *target, *copy;
1723         ucl_object_iter_t it = NULL;
1724         bool replace = false;
1725         struct ucl_parser *parser = ud;
1726
1727         parent = ucl_object_lookup_len (ctx, data, len);
1728
1729         /* Some sanity checks */
1730         if (parent == NULL || ucl_object_type (parent) != UCL_OBJECT) {
1731                 ucl_create_err (&parser->err, "Unable to find inherited object %*.s",
1732                                 (int)len, data);
1733                 return false;
1734         }
1735
1736         if (parser->stack == NULL || parser->stack->obj == NULL ||
1737                         ucl_object_type (parser->stack->obj) != UCL_OBJECT) {
1738                 ucl_create_err (&parser->err, "Invalid inherit context");
1739                 return false;
1740         }
1741
1742         target = parser->stack->obj;
1743
1744         if (args && (cur = ucl_object_lookup (args, "replace")) != NULL) {
1745                 replace = ucl_object_toboolean (cur);
1746         }
1747
1748         while ((cur = ucl_object_iterate (parent, &it, true))) {
1749                 /* We do not replace existing keys */
1750                 if (!replace && ucl_object_lookup_len (target, cur->key, cur->keylen)) {
1751                         continue;
1752                 }
1753
1754                 copy = ucl_object_copy (cur);
1755
1756                 if (!replace) {
1757                         copy->flags |= UCL_OBJECT_INHERITED;
1758                 }
1759
1760                 ucl_object_insert_key (target, copy, copy->key,
1761                                 copy->keylen, false);
1762         }
1763
1764         return true;
1765 }
1766
1767 bool
1768 ucl_parser_set_filevars (struct ucl_parser *parser, const char *filename, bool need_expand)
1769 {
1770         char realbuf[PATH_MAX], *curdir;
1771
1772         if (filename != NULL) {
1773                 if (need_expand) {
1774                         if (ucl_realpath (filename, realbuf) == NULL) {
1775                                 return false;
1776                         }
1777                 }
1778                 else {
1779                         ucl_strlcpy (realbuf, filename, sizeof (realbuf));
1780                 }
1781
1782                 /* Define variables */
1783                 ucl_parser_register_variable (parser, "FILENAME", realbuf);
1784                 curdir = dirname (realbuf);
1785                 ucl_parser_register_variable (parser, "CURDIR", curdir);
1786         }
1787         else {
1788                 /* Set everything from the current dir */
1789                 curdir = getcwd (realbuf, sizeof (realbuf));
1790                 ucl_parser_register_variable (parser, "FILENAME", "undef");
1791                 ucl_parser_register_variable (parser, "CURDIR", curdir);
1792         }
1793
1794         return true;
1795 }
1796
1797 bool
1798 ucl_parser_add_file_full (struct ucl_parser *parser, const char *filename,
1799                 unsigned priority, enum ucl_duplicate_strategy strat,
1800                 enum ucl_parse_type parse_type)
1801 {
1802         unsigned char *buf;
1803         size_t len;
1804         bool ret;
1805         char realbuf[PATH_MAX];
1806
1807         if (ucl_realpath (filename, realbuf) == NULL) {
1808                 ucl_create_err (&parser->err, "cannot open file %s: %s",
1809                                 filename,
1810                                 strerror (errno));
1811                 return false;
1812         }
1813
1814         if (!ucl_fetch_file (realbuf, &buf, &len, &parser->err, true)) {
1815                 return false;
1816         }
1817
1818         if (parser->cur_file) {
1819                 free (parser->cur_file);
1820         }
1821         parser->cur_file = strdup (realbuf);
1822         ucl_parser_set_filevars (parser, realbuf, false);
1823         ret = ucl_parser_add_chunk_full (parser, buf, len, priority, strat,
1824                         parse_type);
1825
1826         if (len > 0) {
1827                 ucl_munmap (buf, len);
1828         }
1829
1830         return ret;
1831 }
1832
1833 bool
1834 ucl_parser_add_file_priority (struct ucl_parser *parser, const char *filename,
1835                 unsigned priority)
1836 {
1837         if (parser == NULL) {
1838                 return false;
1839         }
1840
1841         return ucl_parser_add_file_full(parser, filename, priority,
1842                         UCL_DUPLICATE_APPEND, UCL_PARSE_UCL);
1843 }
1844
1845 bool
1846 ucl_parser_add_file (struct ucl_parser *parser, const char *filename)
1847 {
1848         if (parser == NULL) {
1849                 return false;
1850         }
1851
1852         return ucl_parser_add_file_full(parser, filename,
1853                         parser->default_priority, UCL_DUPLICATE_APPEND,
1854                         UCL_PARSE_UCL);
1855 }
1856
1857 bool
1858 ucl_parser_add_fd_priority (struct ucl_parser *parser, int fd,
1859                 unsigned priority)
1860 {
1861         unsigned char *buf;
1862         size_t len;
1863         bool ret;
1864         struct stat st;
1865
1866         if (fstat (fd, &st) == -1) {
1867                 ucl_create_err (&parser->err, "cannot stat fd %d: %s",
1868                         fd, strerror (errno));
1869                 return false;
1870         }
1871         if (st.st_size == 0) {
1872                 return true;
1873         }
1874         if ((buf = ucl_mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
1875                 ucl_create_err (&parser->err, "cannot mmap fd %d: %s",
1876                         fd, strerror (errno));
1877                 return false;
1878         }
1879
1880         if (parser->cur_file) {
1881                 free (parser->cur_file);
1882         }
1883         parser->cur_file = NULL;
1884         len = st.st_size;
1885         ret = ucl_parser_add_chunk_priority (parser, buf, len, priority);
1886
1887         if (len > 0) {
1888                 ucl_munmap (buf, len);
1889         }
1890
1891         return ret;
1892 }
1893
1894 bool
1895 ucl_parser_add_fd (struct ucl_parser *parser, int fd)
1896 {
1897         if (parser == NULL) {
1898                 return false;
1899         }
1900
1901         return ucl_parser_add_fd_priority(parser, fd, parser->default_priority);
1902 }
1903
1904 size_t
1905 ucl_strlcpy (char *dst, const char *src, size_t siz)
1906 {
1907         char *d = dst;
1908         const char *s = src;
1909         size_t n = siz;
1910
1911         /* Copy as many bytes as will fit */
1912         if (n != 0) {
1913                 while (--n != 0) {
1914                         if ((*d++ = *s++) == '\0') {
1915                                 break;
1916                         }
1917                 }
1918         }
1919
1920         if (n == 0 && siz != 0) {
1921                 *d = '\0';
1922         }
1923
1924         return (s - src - 1);    /* count does not include NUL */
1925 }
1926
1927 size_t
1928 ucl_strlcpy_unsafe (char *dst, const char *src, size_t siz)
1929 {
1930         memcpy (dst, src, siz - 1);
1931         dst[siz - 1] = '\0';
1932
1933         return siz - 1;
1934 }
1935
1936 size_t
1937 ucl_strlcpy_tolower (char *dst, const char *src, size_t siz)
1938 {
1939         char *d = dst;
1940         const char *s = src;
1941         size_t n = siz;
1942
1943         /* Copy as many bytes as will fit */
1944         if (n != 0) {
1945                 while (--n != 0) {
1946                         if ((*d++ = tolower (*s++)) == '\0') {
1947                                 break;
1948                         }
1949                 }
1950         }
1951
1952         if (n == 0 && siz != 0) {
1953                 *d = '\0';
1954         }
1955
1956         return (s - src);    /* count does not include NUL */
1957 }
1958
1959 /*
1960  * Find the first occurrence of find in s
1961  */
1962 char *
1963 ucl_strnstr (const char *s, const char *find, int len)
1964 {
1965         char c, sc;
1966         int mlen;
1967
1968         if ((c = *find++) != 0) {
1969                 mlen = strlen (find);
1970                 do {
1971                         do {
1972                                 if ((sc = *s++) == 0 || len-- == 0)
1973                                         return (NULL);
1974                         } while (sc != c);
1975                 } while (strncmp (s, find, mlen) != 0);
1976                 s--;
1977         }
1978         return ((char *)s);
1979 }
1980
1981 /*
1982  * Find the first occurrence of find in s, ignore case.
1983  */
1984 char *
1985 ucl_strncasestr (const char *s, const char *find, int len)
1986 {
1987         char c, sc;
1988         int mlen;
1989
1990         if ((c = *find++) != 0) {
1991                 c = tolower (c);
1992                 mlen = strlen (find);
1993                 do {
1994                         do {
1995                                 if ((sc = *s++) == 0 || len-- == 0)
1996                                         return (NULL);
1997                         } while (tolower (sc) != c);
1998                 } while (strncasecmp (s, find, mlen) != 0);
1999                 s--;
2000         }
2001         return ((char *)s);
2002 }
2003
2004 ucl_object_t *
2005 ucl_object_fromstring_common (const char *str, size_t len, enum ucl_string_flags flags)
2006 {
2007         ucl_object_t *obj;
2008         const char *start, *end, *p, *pos;
2009         char *dst, *d;
2010         size_t escaped_len;
2011
2012         if (str == NULL) {
2013                 return NULL;
2014         }
2015
2016         obj = ucl_object_new ();
2017         if (obj) {
2018                 if (len == 0) {
2019                         len = strlen (str);
2020                 }
2021                 if (flags & UCL_STRING_TRIM) {
2022                         /* Skip leading spaces */
2023                         for (start = str; (size_t)(start - str) < len; start ++) {
2024                                 if (!ucl_test_character (*start, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
2025                                         break;
2026                                 }
2027                         }
2028                         /* Skip trailing spaces */
2029                         for (end = str + len - 1; end > start; end --) {
2030                                 if (!ucl_test_character (*end, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
2031                                         break;
2032                                 }
2033                         }
2034                         end ++;
2035                 }
2036                 else {
2037                         start = str;
2038                         end = str + len;
2039                 }
2040
2041                 obj->type = UCL_STRING;
2042                 if (flags & UCL_STRING_ESCAPE) {
2043                         for (p = start, escaped_len = 0; p < end; p ++, escaped_len ++) {
2044                                 if (ucl_test_character (*p, UCL_CHARACTER_JSON_UNSAFE)) {
2045                                         escaped_len ++;
2046                                 }
2047                         }
2048                         dst = malloc (escaped_len + 1);
2049                         if (dst != NULL) {
2050                                 for (p = start, d = dst; p < end; p ++, d ++) {
2051                                         if (ucl_test_character (*p, UCL_CHARACTER_JSON_UNSAFE)) {
2052                                                 switch (*p) {
2053                                                 case '\n':
2054                                                         *d++ = '\\';
2055                                                         *d = 'n';
2056                                                         break;
2057                                                 case '\r':
2058                                                         *d++ = '\\';
2059                                                         *d = 'r';
2060                                                         break;
2061                                                 case '\b':
2062                                                         *d++ = '\\';
2063                                                         *d = 'b';
2064                                                         break;
2065                                                 case '\t':
2066                                                         *d++ = '\\';
2067                                                         *d = 't';
2068                                                         break;
2069                                                 case '\f':
2070                                                         *d++ = '\\';
2071                                                         *d = 'f';
2072                                                         break;
2073                                                 case '\\':
2074                                                         *d++ = '\\';
2075                                                         *d = '\\';
2076                                                         break;
2077                                                 case '"':
2078                                                         *d++ = '\\';
2079                                                         *d = '"';
2080                                                         break;
2081                                                 }
2082                                         }
2083                                         else {
2084                                                 *d = *p;
2085                                         }
2086                                 }
2087                                 *d = '\0';
2088                                 obj->value.sv = dst;
2089                                 obj->trash_stack[UCL_TRASH_VALUE] = dst;
2090                                 obj->len = escaped_len;
2091                         }
2092                 }
2093                 else {
2094                         dst = malloc (end - start + 1);
2095                         if (dst != NULL) {
2096                                 ucl_strlcpy_unsafe (dst, start, end - start + 1);
2097                                 obj->value.sv = dst;
2098                                 obj->trash_stack[UCL_TRASH_VALUE] = dst;
2099                                 obj->len = end - start;
2100                         }
2101                 }
2102                 if ((flags & UCL_STRING_PARSE) && dst != NULL) {
2103                         /* Parse what we have */
2104                         if (flags & UCL_STRING_PARSE_BOOLEAN) {
2105                                 if (!ucl_maybe_parse_boolean (obj, dst, obj->len) && (flags & UCL_STRING_PARSE_NUMBER)) {
2106                                         ucl_maybe_parse_number (obj, dst, dst + obj->len, &pos,
2107                                                         flags & UCL_STRING_PARSE_DOUBLE,
2108                                                         flags & UCL_STRING_PARSE_BYTES,
2109                                                         flags & UCL_STRING_PARSE_TIME);
2110                                 }
2111                         }
2112                         else {
2113                                 ucl_maybe_parse_number (obj, dst, dst + obj->len, &pos,
2114                                                 flags & UCL_STRING_PARSE_DOUBLE,
2115                                                 flags & UCL_STRING_PARSE_BYTES,
2116                                                 flags & UCL_STRING_PARSE_TIME);
2117                         }
2118                 }
2119         }
2120
2121         return obj;
2122 }
2123
2124 static bool
2125 ucl_object_insert_key_common (ucl_object_t *top, ucl_object_t *elt,
2126                 const char *key, size_t keylen, bool copy_key, bool merge, bool replace)
2127 {
2128         ucl_object_t *found, *tmp;
2129         const ucl_object_t *cur;
2130         ucl_object_iter_t it = NULL;
2131         const char *p;
2132         int ret = true;
2133
2134         if (elt == NULL || key == NULL) {
2135                 return false;
2136         }
2137
2138         if (top == NULL) {
2139                 return false;
2140         }
2141
2142         if (top->type != UCL_OBJECT) {
2143                 /* It is possible to convert NULL type to an object */
2144                 if (top->type == UCL_NULL) {
2145                         top->type = UCL_OBJECT;
2146                 }
2147                 else {
2148                         /* Refuse converting of other object types */
2149                         return false;
2150                 }
2151         }
2152
2153         if (top->value.ov == NULL) {
2154                 top->value.ov = ucl_hash_create (false);
2155         }
2156
2157         if (keylen == 0) {
2158                 keylen = strlen (key);
2159         }
2160
2161         for (p = key; p < key + keylen; p ++) {
2162                 if (ucl_test_character (*p, UCL_CHARACTER_UCL_UNSAFE)) {
2163                         elt->flags |= UCL_OBJECT_NEED_KEY_ESCAPE;
2164                         break;
2165                 }
2166         }
2167
2168         /* workaround for some use cases */
2169         if (elt->trash_stack[UCL_TRASH_KEY] != NULL &&
2170                         key != (const char *)elt->trash_stack[UCL_TRASH_KEY]) {
2171                 /* Remove copied key */
2172                 free (elt->trash_stack[UCL_TRASH_KEY]);
2173                 elt->trash_stack[UCL_TRASH_KEY] = NULL;
2174                 elt->flags &= ~UCL_OBJECT_ALLOCATED_KEY;
2175         }
2176
2177         elt->key = key;
2178         elt->keylen = keylen;
2179
2180         if (copy_key) {
2181                 ucl_copy_key_trash (elt);
2182         }
2183
2184         found = __DECONST (ucl_object_t *, ucl_hash_search_obj (top->value.ov, elt));
2185
2186         if (found == NULL) {
2187                 top->value.ov = ucl_hash_insert_object (top->value.ov, elt, false);
2188                 top->len ++;
2189                 if (replace) {
2190                         ret = false;
2191                 }
2192         }
2193         else {
2194                 if (replace) {
2195                         ucl_hash_replace (top->value.ov, found, elt);
2196                         ucl_object_unref (found);
2197                 }
2198                 else if (merge) {
2199                         if (found->type != UCL_OBJECT && elt->type == UCL_OBJECT) {
2200                                 /* Insert old elt to new one */
2201                                 ucl_object_insert_key_common (elt, found, found->key,
2202                                                 found->keylen, copy_key, false, false);
2203                                 ucl_hash_delete (top->value.ov, found);
2204                                 top->value.ov = ucl_hash_insert_object (top->value.ov, elt, false);
2205                         }
2206                         else if (found->type == UCL_OBJECT && elt->type != UCL_OBJECT) {
2207                                 /* Insert new to old */
2208                                 ucl_object_insert_key_common (found, elt, elt->key,
2209                                                 elt->keylen, copy_key, false, false);
2210                         }
2211                         else if (found->type == UCL_OBJECT && elt->type == UCL_OBJECT) {
2212                                 /* Mix two hashes */
2213                                 while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) {
2214                                         tmp = ucl_object_ref (cur);
2215                                         ucl_object_insert_key_common (found, tmp, cur->key,
2216                                                         cur->keylen, copy_key, false, false);
2217                                 }
2218                                 ucl_object_unref (elt);
2219                         }
2220                         else {
2221                                 /* Just make a list of scalars */
2222                                 DL_APPEND (found, elt);
2223                         }
2224                 }
2225                 else {
2226                         DL_APPEND (found, elt);
2227                 }
2228         }
2229
2230         return ret;
2231 }
2232
2233 bool
2234 ucl_object_delete_keyl (ucl_object_t *top, const char *key, size_t keylen)
2235 {
2236         ucl_object_t *found;
2237
2238         if (top == NULL || key == NULL) {
2239                 return false;
2240         }
2241
2242         found = __DECONST (ucl_object_t *, ucl_object_lookup_len (top, key, keylen));
2243
2244         if (found == NULL) {
2245                 return false;
2246         }
2247
2248         ucl_hash_delete (top->value.ov, found);
2249         ucl_object_unref (found);
2250         top->len --;
2251
2252         return true;
2253 }
2254
2255 bool
2256 ucl_object_delete_key (ucl_object_t *top, const char *key)
2257 {
2258         return ucl_object_delete_keyl (top, key, strlen (key));
2259 }
2260
2261 ucl_object_t*
2262 ucl_object_pop_keyl (ucl_object_t *top, const char *key, size_t keylen)
2263 {
2264         const ucl_object_t *found;
2265
2266         if (top == NULL || key == NULL) {
2267                 return false;
2268         }
2269         found = ucl_object_lookup_len (top, key, keylen);
2270
2271         if (found == NULL) {
2272                 return NULL;
2273         }
2274         ucl_hash_delete (top->value.ov, found);
2275         top->len --;
2276
2277         return __DECONST (ucl_object_t *, found);
2278 }
2279
2280 ucl_object_t*
2281 ucl_object_pop_key (ucl_object_t *top, const char *key)
2282 {
2283         return ucl_object_pop_keyl (top, key, strlen (key));
2284 }
2285
2286 bool
2287 ucl_object_insert_key (ucl_object_t *top, ucl_object_t *elt,
2288                 const char *key, size_t keylen, bool copy_key)
2289 {
2290         return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, false, false);
2291 }
2292
2293 bool
2294 ucl_object_insert_key_merged (ucl_object_t *top, ucl_object_t *elt,
2295                 const char *key, size_t keylen, bool copy_key)
2296 {
2297         return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, true, false);
2298 }
2299
2300 bool
2301 ucl_object_replace_key (ucl_object_t *top, ucl_object_t *elt,
2302                 const char *key, size_t keylen, bool copy_key)
2303 {
2304         return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, false, true);
2305 }
2306
2307 bool
2308 ucl_object_merge (ucl_object_t *top, ucl_object_t *elt, bool copy)
2309 {
2310         ucl_object_t *cur = NULL, *cp = NULL, *found = NULL;
2311         ucl_object_iter_t iter = NULL;
2312
2313         if (top == NULL || top->type != UCL_OBJECT || elt == NULL || elt->type != UCL_OBJECT) {
2314                 return false;
2315         }
2316
2317         /* Mix two hashes */
2318         while ((cur = (ucl_object_t*)ucl_hash_iterate (elt->value.ov, &iter))) {
2319                 if (copy) {
2320                         cp = ucl_object_copy (cur);
2321                 }
2322                 else {
2323                         cp = ucl_object_ref (cur);
2324                 }
2325                 found = __DECONST(ucl_object_t *, ucl_hash_search (top->value.ov, cp->key, cp->keylen));
2326                 if (found == NULL) {
2327                         /* The key does not exist */
2328                         top->value.ov = ucl_hash_insert_object (top->value.ov, cp, false);
2329                         top->len ++;
2330                 }
2331                 else {
2332                         /* The key already exists, replace it */
2333                         ucl_hash_replace (top->value.ov, found, cp);
2334                         ucl_object_unref (found);
2335                 }
2336         }
2337
2338         return true;
2339 }
2340
2341 const ucl_object_t *
2342 ucl_object_lookup_len (const ucl_object_t *obj, const char *key, size_t klen)
2343 {
2344         const ucl_object_t *ret;
2345         ucl_object_t srch;
2346
2347         if (obj == NULL || obj->type != UCL_OBJECT || key == NULL) {
2348                 return NULL;
2349         }
2350
2351         srch.key = key;
2352         srch.keylen = klen;
2353         ret = ucl_hash_search_obj (obj->value.ov, &srch);
2354
2355         return ret;
2356 }
2357
2358 const ucl_object_t *
2359 ucl_object_lookup (const ucl_object_t *obj, const char *key)
2360 {
2361         if (key == NULL) {
2362                 return NULL;
2363         }
2364
2365         return ucl_object_lookup_len (obj, key, strlen (key));
2366 }
2367
2368 const ucl_object_t*
2369 ucl_object_lookup_any (const ucl_object_t *obj,
2370                 const char *key, ...)
2371 {
2372         va_list ap;
2373         const ucl_object_t *ret = NULL;
2374         const char *nk = NULL;
2375
2376         if (obj == NULL || key == NULL) {
2377                 return NULL;
2378         }
2379
2380         ret = ucl_object_lookup_len (obj, key, strlen (key));
2381
2382         if (ret == NULL) {
2383                 va_start (ap, key);
2384
2385                 while (ret == NULL) {
2386                         nk = va_arg (ap, const char *);
2387
2388                         if (nk == NULL) {
2389                                 break;
2390                         }
2391                         else {
2392                                 ret = ucl_object_lookup_len (obj, nk, strlen (nk));
2393                         }
2394                 }
2395
2396                 va_end (ap);
2397         }
2398
2399         return ret;
2400 }
2401
2402 const ucl_object_t*
2403 ucl_object_iterate (const ucl_object_t *obj, ucl_object_iter_t *iter, bool expand_values)
2404 {
2405         const ucl_object_t *elt = NULL;
2406
2407         if (obj == NULL || iter == NULL) {
2408                 return NULL;
2409         }
2410
2411         if (expand_values) {
2412                 switch (obj->type) {
2413                 case UCL_OBJECT:
2414                         return (const ucl_object_t*)ucl_hash_iterate (obj->value.ov, iter);
2415                         break;
2416                 case UCL_ARRAY: {
2417                         unsigned int idx;
2418                         UCL_ARRAY_GET (vec, obj);
2419                         idx = (unsigned int)(uintptr_t)(*iter);
2420
2421                         if (vec != NULL) {
2422                                 while (idx < kv_size (*vec)) {
2423                                         if ((elt = kv_A (*vec, idx)) != NULL) {
2424                                                 idx ++;
2425                                                 break;
2426                                         }
2427                                         idx ++;
2428                                 }
2429                                 *iter = (void *)(uintptr_t)idx;
2430                         }
2431
2432                         return elt;
2433                         break;
2434                 }
2435                 default:
2436                         /* Go to linear iteration */
2437                         break;
2438                 }
2439         }
2440         /* Treat everything as a linear list */
2441         elt = *iter;
2442         if (elt == NULL) {
2443                 elt = obj;
2444         }
2445         else if (elt == obj) {
2446                 return NULL;
2447         }
2448         *iter = __DECONST (void *, elt->next ? elt->next : obj);
2449         return elt;
2450
2451         /* Not reached */
2452         return NULL;
2453 }
2454
2455 const char safe_iter_magic[4] = {'u', 'i', 't', 'e'};
2456 struct ucl_object_safe_iter {
2457         char magic[4]; /* safety check */
2458         const ucl_object_t *impl_it; /* implicit object iteration */
2459         ucl_object_iter_t expl_it; /* explicit iteration */
2460 };
2461
2462 #define UCL_SAFE_ITER(ptr) (struct ucl_object_safe_iter *)(ptr)
2463 #define UCL_SAFE_ITER_CHECK(it) do { \
2464         assert (it != NULL); \
2465         assert (memcmp (it->magic, safe_iter_magic, sizeof (it->magic)) == 0); \
2466  } while (0)
2467
2468 ucl_object_iter_t
2469 ucl_object_iterate_new (const ucl_object_t *obj)
2470 {
2471         struct ucl_object_safe_iter *it;
2472
2473         it = UCL_ALLOC (sizeof (*it));
2474         if (it != NULL) {
2475                 memcpy (it->magic, safe_iter_magic, sizeof (it->magic));
2476                 it->expl_it = NULL;
2477                 it->impl_it = obj;
2478         }
2479
2480         return (ucl_object_iter_t)it;
2481 }
2482
2483
2484 ucl_object_iter_t
2485 ucl_object_iterate_reset (ucl_object_iter_t it, const ucl_object_t *obj)
2486 {
2487         struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
2488
2489         UCL_SAFE_ITER_CHECK (rit);
2490
2491         rit->impl_it = obj;
2492         rit->expl_it = NULL;
2493
2494         return it;
2495 }
2496
2497 const ucl_object_t*
2498 ucl_object_iterate_safe (ucl_object_iter_t it, bool expand_values)
2499 {
2500         struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
2501         const ucl_object_t *ret = NULL;
2502
2503         UCL_SAFE_ITER_CHECK (rit);
2504
2505         if (rit->impl_it == NULL) {
2506                 return NULL;
2507         }
2508
2509         if (rit->impl_it->type == UCL_OBJECT || rit->impl_it->type == UCL_ARRAY) {
2510                 ret = ucl_object_iterate (rit->impl_it, &rit->expl_it, true);
2511
2512                 if (ret == NULL) {
2513                         /* Need to switch to another implicit object in chain */
2514                         rit->impl_it = rit->impl_it->next;
2515                         rit->expl_it = NULL;
2516                         return ucl_object_iterate_safe (it, expand_values);
2517                 }
2518         }
2519         else {
2520                 /* Just iterate over the implicit array */
2521                 ret = rit->impl_it;
2522                 rit->impl_it = rit->impl_it->next;
2523                 if (expand_values) {
2524                         /* We flatten objects if need to expand values */
2525                         if (ret->type == UCL_OBJECT || ret->type == UCL_ARRAY) {
2526                                 return ucl_object_iterate_safe (it, expand_values);
2527                         }
2528                 }
2529         }
2530
2531         return ret;
2532 }
2533
2534 void
2535 ucl_object_iterate_free (ucl_object_iter_t it)
2536 {
2537         struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
2538
2539         UCL_SAFE_ITER_CHECK (rit);
2540
2541         UCL_FREE (sizeof (*rit), it);
2542 }
2543
2544 const ucl_object_t *
2545 ucl_object_lookup_path (const ucl_object_t *top, const char *path_in) {
2546         return ucl_object_lookup_path_char (top, path_in, '.');
2547 }
2548
2549
2550 const ucl_object_t *
2551 ucl_object_lookup_path_char (const ucl_object_t *top, const char *path_in, const char sep) {
2552         const ucl_object_t *o = NULL, *found;
2553         const char *p, *c;
2554         char *err_str;
2555         unsigned index;
2556
2557         if (path_in == NULL || top == NULL) {
2558                 return NULL;
2559         }
2560
2561         found = NULL;
2562         p = path_in;
2563
2564         /* Skip leading dots */
2565         while (*p == sep) {
2566                 p ++;
2567         }
2568
2569         c = p;
2570         while (*p != '\0') {
2571                 p ++;
2572                 if (*p == sep || *p == '\0') {
2573                         if (p > c) {
2574                                 switch (top->type) {
2575                                 case UCL_ARRAY:
2576                                         /* Key should be an int */
2577                                         index = strtoul (c, &err_str, 10);
2578                                         if (err_str != NULL && (*err_str != sep && *err_str != '\0')) {
2579                                                 return NULL;
2580                                         }
2581                                         o = ucl_array_find_index (top, index);
2582                                         break;
2583                                 default:
2584                                         o = ucl_object_lookup_len (top, c, p - c);
2585                                         break;
2586                                 }
2587                                 if (o == NULL) {
2588                                         return NULL;
2589                                 }
2590                                 top = o;
2591                         }
2592                         if (*p != '\0') {
2593                                 c = p + 1;
2594                         }
2595                 }
2596         }
2597         found = o;
2598
2599         return found;
2600 }
2601
2602
2603 ucl_object_t *
2604 ucl_object_new (void)
2605 {
2606         return ucl_object_typed_new (UCL_NULL);
2607 }
2608
2609 ucl_object_t *
2610 ucl_object_typed_new (ucl_type_t type)
2611 {
2612         return ucl_object_new_full (type, 0);
2613 }
2614
2615 ucl_object_t *
2616 ucl_object_new_full (ucl_type_t type, unsigned priority)
2617 {
2618         ucl_object_t *new;
2619
2620         if (type != UCL_USERDATA) {
2621                 new = UCL_ALLOC (sizeof (ucl_object_t));
2622                 if (new != NULL) {
2623                         memset (new, 0, sizeof (ucl_object_t));
2624                         new->ref = 1;
2625                         new->type = (type <= UCL_NULL ? type : UCL_NULL);
2626                         new->next = NULL;
2627                         new->prev = new;
2628                         ucl_object_set_priority (new, priority);
2629
2630                         if (type == UCL_ARRAY) {
2631                                 new->value.av = UCL_ALLOC (sizeof (ucl_array_t));
2632                                 if (new->value.av) {
2633                                         memset (new->value.av, 0, sizeof (ucl_array_t));
2634                                         UCL_ARRAY_GET (vec, new);
2635
2636                                         /* Preallocate some space for arrays */
2637                                         kv_resize (ucl_object_t *, *vec, 8);
2638                                 }
2639                         }
2640                 }
2641         }
2642         else {
2643                 new = ucl_object_new_userdata (NULL, NULL, NULL);
2644                 ucl_object_set_priority (new, priority);
2645         }
2646
2647         return new;
2648 }
2649
2650 ucl_object_t*
2651 ucl_object_new_userdata (ucl_userdata_dtor dtor,
2652                 ucl_userdata_emitter emitter,
2653                 void *ptr)
2654 {
2655         struct ucl_object_userdata *new;
2656         size_t nsize = sizeof (*new);
2657
2658         new = UCL_ALLOC (nsize);
2659         if (new != NULL) {
2660                 memset (new, 0, nsize);
2661                 new->obj.ref = 1;
2662                 new->obj.type = UCL_USERDATA;
2663                 new->obj.next = NULL;
2664                 new->obj.prev = (ucl_object_t *)new;
2665                 new->dtor = dtor;
2666                 new->emitter = emitter;
2667                 new->obj.value.ud = ptr;
2668         }
2669
2670         return (ucl_object_t *)new;
2671 }
2672
2673 ucl_type_t
2674 ucl_object_type (const ucl_object_t *obj)
2675 {
2676         if (obj == NULL) {
2677                 return UCL_NULL;
2678         }
2679
2680         return obj->type;
2681 }
2682
2683 ucl_object_t*
2684 ucl_object_fromstring (const char *str)
2685 {
2686         return ucl_object_fromstring_common (str, 0, UCL_STRING_ESCAPE);
2687 }
2688
2689 ucl_object_t *
2690 ucl_object_fromlstring (const char *str, size_t len)
2691 {
2692         return ucl_object_fromstring_common (str, len, UCL_STRING_ESCAPE);
2693 }
2694
2695 ucl_object_t *
2696 ucl_object_fromint (int64_t iv)
2697 {
2698         ucl_object_t *obj;
2699
2700         obj = ucl_object_new ();
2701         if (obj != NULL) {
2702                 obj->type = UCL_INT;
2703                 obj->value.iv = iv;
2704         }
2705
2706         return obj;
2707 }
2708
2709 ucl_object_t *
2710 ucl_object_fromdouble (double dv)
2711 {
2712         ucl_object_t *obj;
2713
2714         obj = ucl_object_new ();
2715         if (obj != NULL) {
2716                 obj->type = UCL_FLOAT;
2717                 obj->value.dv = dv;
2718         }
2719
2720         return obj;
2721 }
2722
2723 ucl_object_t*
2724 ucl_object_frombool (bool bv)
2725 {
2726         ucl_object_t *obj;
2727
2728         obj = ucl_object_new ();
2729         if (obj != NULL) {
2730                 obj->type = UCL_BOOLEAN;
2731                 obj->value.iv = bv;
2732         }
2733
2734         return obj;
2735 }
2736
2737 bool
2738 ucl_array_append (ucl_object_t *top, ucl_object_t *elt)
2739 {
2740         UCL_ARRAY_GET (vec, top);
2741
2742         if (elt == NULL || top == NULL) {
2743                 return false;
2744         }
2745
2746         if (vec == NULL) {
2747                 vec = UCL_ALLOC (sizeof (*vec));
2748
2749                 if (vec == NULL) {
2750                         return false;
2751                 }
2752
2753                 kv_init (*vec);
2754                 top->value.av = (void *)vec;
2755         }
2756
2757         kv_push (ucl_object_t *, *vec, elt);
2758
2759         top->len ++;
2760
2761         return true;
2762 }
2763
2764 bool
2765 ucl_array_prepend (ucl_object_t *top, ucl_object_t *elt)
2766 {
2767         UCL_ARRAY_GET (vec, top);
2768
2769         if (elt == NULL || top == NULL) {
2770                 return false;
2771         }
2772
2773         if (vec == NULL) {
2774                 vec = UCL_ALLOC (sizeof (*vec));
2775                 kv_init (*vec);
2776                 top->value.av = (void *)vec;
2777                 kv_push (ucl_object_t *, *vec, elt);
2778         }
2779         else {
2780                 /* Slow O(n) algorithm */
2781                 kv_prepend (ucl_object_t *, *vec, elt);
2782         }
2783
2784         top->len ++;
2785
2786         return true;
2787 }
2788
2789 bool
2790 ucl_array_merge (ucl_object_t *top, ucl_object_t *elt, bool copy)
2791 {
2792         unsigned i;
2793         ucl_object_t *cp = NULL;
2794         ucl_object_t **obj;
2795
2796         if (elt == NULL || top == NULL || top->type != UCL_ARRAY || elt->type != UCL_ARRAY) {
2797                 return false;
2798         }
2799
2800         if (copy) {
2801                 cp = ucl_object_copy (elt);
2802         }
2803         else {
2804                 cp = ucl_object_ref (elt);
2805         }
2806
2807         UCL_ARRAY_GET (v1, top);
2808         UCL_ARRAY_GET (v2, cp);
2809
2810         if (v1 && v2) {
2811                 kv_concat (ucl_object_t *, *v1, *v2);
2812
2813                 for (i = v2->n; i < v1->n; i ++) {
2814                         obj = &kv_A (*v1, i);
2815                         if (*obj == NULL) {
2816                                 continue;
2817                         }
2818                         top->len ++;
2819                 }
2820         }
2821
2822         return true;
2823 }
2824
2825 ucl_object_t *
2826 ucl_array_delete (ucl_object_t *top, ucl_object_t *elt)
2827 {
2828         UCL_ARRAY_GET (vec, top);
2829         ucl_object_t *ret = NULL;
2830         unsigned i;
2831
2832         if (vec == NULL) {
2833                 return NULL;
2834         }
2835
2836         for (i = 0; i < vec->n; i ++) {
2837                 if (kv_A (*vec, i) == elt) {
2838                         kv_del (ucl_object_t *, *vec, i);
2839                         ret = elt;
2840                         top->len --;
2841                         break;
2842                 }
2843         }
2844
2845         return ret;
2846 }
2847
2848 const ucl_object_t *
2849 ucl_array_head (const ucl_object_t *top)
2850 {
2851         UCL_ARRAY_GET (vec, top);
2852
2853         if (vec == NULL || top == NULL || top->type != UCL_ARRAY ||
2854                         top->value.av == NULL) {
2855                 return NULL;
2856         }
2857
2858         return (vec->n > 0 ? vec->a[0] : NULL);
2859 }
2860
2861 const ucl_object_t *
2862 ucl_array_tail (const ucl_object_t *top)
2863 {
2864         UCL_ARRAY_GET (vec, top);
2865
2866         if (top == NULL || top->type != UCL_ARRAY || top->value.av == NULL) {
2867                 return NULL;
2868         }
2869
2870         return (vec->n > 0 ? vec->a[vec->n - 1] : NULL);
2871 }
2872
2873 ucl_object_t *
2874 ucl_array_pop_last (ucl_object_t *top)
2875 {
2876         UCL_ARRAY_GET (vec, top);
2877         ucl_object_t **obj, *ret = NULL;
2878
2879         if (vec != NULL && vec->n > 0) {
2880                 obj = &kv_A (*vec, vec->n - 1);
2881                 ret = *obj;
2882                 kv_del (ucl_object_t *, *vec, vec->n - 1);
2883                 top->len --;
2884         }
2885
2886         return ret;
2887 }
2888
2889 ucl_object_t *
2890 ucl_array_pop_first (ucl_object_t *top)
2891 {
2892         UCL_ARRAY_GET (vec, top);
2893         ucl_object_t **obj, *ret = NULL;
2894
2895         if (vec != NULL && vec->n > 0) {
2896                 obj = &kv_A (*vec, 0);
2897                 ret = *obj;
2898                 kv_del (ucl_object_t *, *vec, 0);
2899                 top->len --;
2900         }
2901
2902         return ret;
2903 }
2904
2905 const ucl_object_t *
2906 ucl_array_find_index (const ucl_object_t *top, unsigned int index)
2907 {
2908         UCL_ARRAY_GET (vec, top);
2909
2910         if (vec != NULL && vec->n > 0 && index < vec->n) {
2911                 return kv_A (*vec, index);
2912         }
2913
2914         return NULL;
2915 }
2916
2917 unsigned int
2918 ucl_array_index_of (ucl_object_t *top, ucl_object_t *elt)
2919 {
2920         UCL_ARRAY_GET (vec, top);
2921         unsigned i;
2922
2923         if (vec == NULL) {
2924                 return (unsigned int)(-1);
2925         }
2926
2927         for (i = 0; i < vec->n; i ++) {
2928                 if (kv_A (*vec, i) == elt) {
2929                         return i;
2930                 }
2931         }
2932
2933         return (unsigned int)(-1);
2934 }
2935
2936 ucl_object_t *
2937 ucl_array_replace_index (ucl_object_t *top, ucl_object_t *elt,
2938         unsigned int index)
2939 {
2940         UCL_ARRAY_GET (vec, top);
2941         ucl_object_t *ret = NULL;
2942
2943         if (vec != NULL && vec->n > 0 && index < vec->n) {
2944                 ret = kv_A (*vec, index);
2945                 kv_A (*vec, index) = elt;
2946         }
2947
2948         return ret;
2949 }
2950
2951 ucl_object_t *
2952 ucl_elt_append (ucl_object_t *head, ucl_object_t *elt)
2953 {
2954
2955         if (head == NULL) {
2956                 elt->next = NULL;
2957                 elt->prev = elt;
2958                 head = elt;
2959         }
2960         else {
2961                 elt->prev = head->prev;
2962                 head->prev->next = elt;
2963                 head->prev = elt;
2964                 elt->next = NULL;
2965         }
2966
2967         return head;
2968 }
2969
2970 bool
2971 ucl_object_todouble_safe (const ucl_object_t *obj, double *target)
2972 {
2973         if (obj == NULL || target == NULL) {
2974                 return false;
2975         }
2976         switch (obj->type) {
2977         case UCL_INT:
2978                 *target = obj->value.iv; /* Probaly could cause overflow */
2979                 break;
2980         case UCL_FLOAT:
2981         case UCL_TIME:
2982                 *target = obj->value.dv;
2983                 break;
2984         default:
2985                 return false;
2986         }
2987
2988         return true;
2989 }
2990
2991 double
2992 ucl_object_todouble (const ucl_object_t *obj)
2993 {
2994         double result = 0.;
2995
2996         ucl_object_todouble_safe (obj, &result);
2997         return result;
2998 }
2999
3000 bool
3001 ucl_object_toint_safe (const ucl_object_t *obj, int64_t *target)
3002 {
3003         if (obj == NULL || target == NULL) {
3004                 return false;
3005         }
3006         switch (obj->type) {
3007         case UCL_INT:
3008                 *target = obj->value.iv;
3009                 break;
3010         case UCL_FLOAT:
3011         case UCL_TIME:
3012                 *target = obj->value.dv; /* Loosing of decimal points */
3013                 break;
3014         default:
3015                 return false;
3016         }
3017
3018         return true;
3019 }
3020
3021 int64_t
3022 ucl_object_toint (const ucl_object_t *obj)
3023 {
3024         int64_t result = 0;
3025
3026         ucl_object_toint_safe (obj, &result);
3027         return result;
3028 }
3029
3030 bool
3031 ucl_object_toboolean_safe (const ucl_object_t *obj, bool *target)
3032 {
3033         if (obj == NULL || target == NULL) {
3034                 return false;
3035         }
3036         switch (obj->type) {
3037         case UCL_BOOLEAN:
3038                 *target = (obj->value.iv == true);
3039                 break;
3040         default:
3041                 return false;
3042         }
3043
3044         return true;
3045 }
3046
3047 bool
3048 ucl_object_toboolean (const ucl_object_t *obj)
3049 {
3050         bool result = false;
3051
3052         ucl_object_toboolean_safe (obj, &result);
3053         return result;
3054 }
3055
3056 bool
3057 ucl_object_tostring_safe (const ucl_object_t *obj, const char **target)
3058 {
3059         if (obj == NULL || target == NULL) {
3060                 return false;
3061         }
3062
3063         switch (obj->type) {
3064         case UCL_STRING:
3065                 if (!(obj->flags & UCL_OBJECT_BINARY)) {
3066                         *target = ucl_copy_value_trash (obj);
3067                 }
3068                 break;
3069         default:
3070                 return false;
3071         }
3072
3073         return true;
3074 }
3075
3076 const char *
3077 ucl_object_tostring (const ucl_object_t *obj)
3078 {
3079         const char *result = NULL;
3080
3081         ucl_object_tostring_safe (obj, &result);
3082         return result;
3083 }
3084
3085 const char *
3086 ucl_object_tostring_forced (const ucl_object_t *obj)
3087 {
3088         /* TODO: For binary strings we might encode string here */
3089         if (!(obj->flags & UCL_OBJECT_BINARY)) {
3090                 return ucl_copy_value_trash (obj);
3091         }
3092
3093         return NULL;
3094 }
3095
3096 bool
3097 ucl_object_tolstring_safe (const ucl_object_t *obj, const char **target, size_t *tlen)
3098 {
3099         if (obj == NULL || target == NULL) {
3100                 return false;
3101         }
3102         switch (obj->type) {
3103         case UCL_STRING:
3104                 *target = obj->value.sv;
3105                 if (tlen != NULL) {
3106                         *tlen = obj->len;
3107                 }
3108                 break;
3109         default:
3110                 return false;
3111         }
3112
3113         return true;
3114 }
3115
3116 const char *
3117 ucl_object_tolstring (const ucl_object_t *obj, size_t *tlen)
3118 {
3119         const char *result = NULL;
3120
3121         ucl_object_tolstring_safe (obj, &result, tlen);
3122         return result;
3123 }
3124
3125 const char *
3126 ucl_object_key (const ucl_object_t *obj)
3127 {
3128         return ucl_copy_key_trash (obj);
3129 }
3130
3131 const char *
3132 ucl_object_keyl (const ucl_object_t *obj, size_t *len)
3133 {
3134         if (len == NULL || obj == NULL) {
3135                 return NULL;
3136         }
3137         *len = obj->keylen;
3138         return obj->key;
3139 }
3140
3141 ucl_object_t *
3142 ucl_object_ref (const ucl_object_t *obj)
3143 {
3144         ucl_object_t *res = NULL;
3145
3146         if (obj != NULL) {
3147                 if (obj->flags & UCL_OBJECT_EPHEMERAL) {
3148                         /*
3149                          * Use deep copy for ephemeral objects, note that its refcount
3150                          * is NOT increased, since ephemeral objects does not need refcount
3151                          * at all
3152                          */
3153                         res = ucl_object_copy (obj);
3154                 }
3155                 else {
3156                         res = __DECONST (ucl_object_t *, obj);
3157 #ifdef HAVE_ATOMIC_BUILTINS
3158                         (void)__sync_add_and_fetch (&res->ref, 1);
3159 #else
3160                         res->ref ++;
3161 #endif
3162                 }
3163         }
3164         return res;
3165 }
3166
3167 static ucl_object_t *
3168 ucl_object_copy_internal (const ucl_object_t *other, bool allow_array)
3169 {
3170
3171         ucl_object_t *new;
3172         ucl_object_iter_t it = NULL;
3173         const ucl_object_t *cur;
3174
3175         new = malloc (sizeof (*new));
3176
3177         if (new != NULL) {
3178                 memcpy (new, other, sizeof (*new));
3179                 if (other->flags & UCL_OBJECT_EPHEMERAL) {
3180                         /* Copied object is always non ephemeral */
3181                         new->flags &= ~UCL_OBJECT_EPHEMERAL;
3182                 }
3183                 new->ref = 1;
3184                 /* Unlink from others */
3185                 new->next = NULL;
3186                 new->prev = new;
3187
3188                 /* deep copy of values stored */
3189                 if (other->trash_stack[UCL_TRASH_KEY] != NULL) {
3190                         new->trash_stack[UCL_TRASH_KEY] =
3191                                         strdup (other->trash_stack[UCL_TRASH_KEY]);
3192                         if (other->key == (const char *)other->trash_stack[UCL_TRASH_KEY]) {
3193                                 new->key = new->trash_stack[UCL_TRASH_KEY];
3194                         }
3195                 }
3196                 if (other->trash_stack[UCL_TRASH_VALUE] != NULL) {
3197                         new->trash_stack[UCL_TRASH_VALUE] =
3198                                         strdup (other->trash_stack[UCL_TRASH_VALUE]);
3199                         if (new->type == UCL_STRING) {
3200                                 new->value.sv = new->trash_stack[UCL_TRASH_VALUE];
3201                         }
3202                 }
3203
3204                 if (other->type == UCL_ARRAY || other->type == UCL_OBJECT) {
3205                         /* reset old value */
3206                         memset (&new->value, 0, sizeof (new->value));
3207
3208                         while ((cur = ucl_object_iterate (other, &it, true)) != NULL) {
3209                                 if (other->type == UCL_ARRAY) {
3210                                         ucl_array_append (new, ucl_object_copy_internal (cur, false));
3211                                 }
3212                                 else {
3213                                         ucl_object_t *cp = ucl_object_copy_internal (cur, true);
3214                                         if (cp != NULL) {
3215                                                 ucl_object_insert_key (new, cp, cp->key, cp->keylen,
3216                                                                 false);
3217                                         }
3218                                 }
3219                         }
3220                 }
3221                 else if (allow_array && other->next != NULL) {
3222                         LL_FOREACH (other->next, cur) {
3223                                 ucl_object_t *cp = ucl_object_copy_internal (cur, false);
3224                                 if (cp != NULL) {
3225                                         DL_APPEND (new, cp);
3226                                 }
3227                         }
3228                 }
3229         }
3230
3231         return new;
3232 }
3233
3234 ucl_object_t *
3235 ucl_object_copy (const ucl_object_t *other)
3236 {
3237         return ucl_object_copy_internal (other, true);
3238 }
3239
3240 void
3241 ucl_object_unref (ucl_object_t *obj)
3242 {
3243         if (obj != NULL) {
3244 #ifdef HAVE_ATOMIC_BUILTINS
3245                 unsigned int rc = __sync_sub_and_fetch (&obj->ref, 1);
3246                 if (rc == 0) {
3247 #else
3248                 if (--obj->ref == 0) {
3249 #endif
3250                         ucl_object_free_internal (obj, true, ucl_object_dtor_unref);
3251                 }
3252         }
3253 }
3254
3255 int
3256 ucl_object_compare (const ucl_object_t *o1, const ucl_object_t *o2)
3257 {
3258         const ucl_object_t *it1, *it2;
3259         ucl_object_iter_t iter = NULL;
3260         int ret = 0;
3261
3262         if (o1->type != o2->type) {
3263                 return (o1->type) - (o2->type);
3264         }
3265
3266         switch (o1->type) {
3267         case UCL_STRING:
3268                 if (o1->len == o2->len && o1->len > 0) {
3269                         ret = strcmp (ucl_object_tostring(o1), ucl_object_tostring(o2));
3270                 }
3271                 else {
3272                         ret = o1->len - o2->len;
3273                 }
3274                 break;
3275         case UCL_FLOAT:
3276         case UCL_INT:
3277         case UCL_TIME:
3278                 ret = ucl_object_todouble (o1) - ucl_object_todouble (o2);
3279                 break;
3280         case UCL_BOOLEAN:
3281                 ret = ucl_object_toboolean (o1) - ucl_object_toboolean (o2);
3282                 break;
3283         case UCL_ARRAY:
3284                 if (o1->len == o2->len && o1->len > 0) {
3285                         UCL_ARRAY_GET (vec1, o1);
3286                         UCL_ARRAY_GET (vec2, o2);
3287                         unsigned i;
3288
3289                         /* Compare all elements in both arrays */
3290                         for (i = 0; i < vec1->n; i ++) {
3291                                 it1 = kv_A (*vec1, i);
3292                                 it2 = kv_A (*vec2, i);
3293
3294                                 if (it1 == NULL && it2 != NULL) {
3295                                         return -1;
3296                                 }
3297                                 else if (it2 == NULL && it1 != NULL) {
3298                                         return 1;
3299                                 }
3300                                 else if (it1 != NULL && it2 != NULL) {
3301                                         ret = ucl_object_compare (it1, it2);
3302                                         if (ret != 0) {
3303                                                 break;
3304                                         }
3305                                 }
3306                         }
3307                 }
3308                 else {
3309                         ret = o1->len - o2->len;
3310                 }
3311                 break;
3312         case UCL_OBJECT:
3313                 if (o1->len == o2->len && o1->len > 0) {
3314                         while ((it1 = ucl_object_iterate (o1, &iter, true)) != NULL) {
3315                                 it2 = ucl_object_lookup (o2, ucl_object_key (it1));
3316                                 if (it2 == NULL) {
3317                                         ret = 1;
3318                                         break;
3319                                 }
3320                                 ret = ucl_object_compare (it1, it2);
3321                                 if (ret != 0) {
3322                                         break;
3323                                 }
3324                         }
3325                 }
3326                 else {
3327                         ret = o1->len - o2->len;
3328                 }
3329                 break;
3330         default:
3331                 ret = 0;
3332                 break;
3333         }
3334
3335         return ret;
3336 }
3337
3338 int
3339 ucl_object_compare_qsort (const ucl_object_t **o1,
3340                 const ucl_object_t **o2)
3341 {
3342         return ucl_object_compare (*o1, *o2);
3343 }
3344
3345 void
3346 ucl_object_array_sort (ucl_object_t *ar,
3347                 int (*cmp)(const ucl_object_t **o1, const ucl_object_t **o2))
3348 {
3349         UCL_ARRAY_GET (vec, ar);
3350
3351         if (cmp == NULL || ar == NULL || ar->type != UCL_ARRAY) {
3352                 return;
3353         }
3354
3355         qsort (vec->a, vec->n, sizeof (ucl_object_t *),
3356                         (int (*)(const void *, const void *))cmp);
3357 }
3358
3359 #define PRIOBITS 4
3360
3361 unsigned int
3362 ucl_object_get_priority (const ucl_object_t *obj)
3363 {
3364         if (obj == NULL) {
3365                 return 0;
3366         }
3367
3368         return (obj->flags >> ((sizeof (obj->flags) * NBBY) - PRIOBITS));
3369 }
3370
3371 void
3372 ucl_object_set_priority (ucl_object_t *obj,
3373                 unsigned int priority)
3374 {
3375         if (obj != NULL) {
3376                 priority &= (0x1 << PRIOBITS) - 1;
3377                 priority <<= ((sizeof (obj->flags) * NBBY) - PRIOBITS);
3378                 priority |= obj->flags & ((1 << ((sizeof (obj->flags) * NBBY) -
3379                                 PRIOBITS)) - 1);
3380                 obj->flags = priority;
3381         }
3382 }
3383
3384 bool
3385 ucl_object_string_to_type (const char *input, ucl_type_t *res)
3386 {
3387         if (strcasecmp (input, "object") == 0) {
3388                 *res = UCL_OBJECT;
3389         }
3390         else if (strcasecmp (input, "array") == 0) {
3391                 *res = UCL_ARRAY;
3392         }
3393         else if (strcasecmp (input, "integer") == 0) {
3394                 *res = UCL_INT;
3395         }
3396         else if (strcasecmp (input, "number") == 0) {
3397                 *res = UCL_FLOAT;
3398         }
3399         else if (strcasecmp (input, "string") == 0) {
3400                 *res = UCL_STRING;
3401         }
3402         else if (strcasecmp (input, "boolean") == 0) {
3403                 *res = UCL_BOOLEAN;
3404         }
3405         else if (strcasecmp (input, "null") == 0) {
3406                 *res = UCL_NULL;
3407         }
3408         else if (strcasecmp (input, "userdata") == 0) {
3409                 *res = UCL_USERDATA;
3410         }
3411         else {
3412                 return false;
3413         }
3414
3415         return true;
3416 }
3417
3418 const char *
3419 ucl_object_type_to_string (ucl_type_t type)
3420 {
3421         const char *res = "unknown";
3422
3423         switch (type) {
3424         case UCL_OBJECT:
3425                 res = "object";
3426                 break;
3427         case UCL_ARRAY:
3428                 res = "array";
3429                 break;
3430         case UCL_INT:
3431                 res = "integer";
3432                 break;
3433         case UCL_FLOAT:
3434         case UCL_TIME:
3435                 res = "number";
3436                 break;
3437         case UCL_STRING:
3438                 res = "string";
3439                 break;
3440         case UCL_BOOLEAN:
3441                 res = "boolean";
3442                 break;
3443         case UCL_USERDATA:
3444                 res = "userdata";
3445                 break;
3446         case UCL_NULL:
3447                 res = "null";
3448                 break;
3449         }
3450
3451         return res;
3452 }
3453
3454 const ucl_object_t *
3455 ucl_parser_get_comments (struct ucl_parser *parser)
3456 {
3457         if (parser && parser->comments) {
3458                 return parser->comments;
3459         }
3460
3461         return NULL;
3462 }
3463
3464 const ucl_object_t *
3465 ucl_comments_find (const ucl_object_t *comments,
3466                 const ucl_object_t *srch)
3467 {
3468         if (comments && srch) {
3469                 return ucl_object_lookup_len (comments, (const char *)&srch,
3470                                 sizeof (void *));
3471         }
3472
3473         return NULL;
3474 }
3475
3476 bool
3477 ucl_comments_move (ucl_object_t *comments,
3478                 const ucl_object_t *from, const ucl_object_t *to)
3479 {
3480         const ucl_object_t *found;
3481         ucl_object_t *obj;
3482
3483         if (comments && from && to) {
3484                 found = ucl_object_lookup_len (comments,
3485                                 (const char *)&from, sizeof (void *));
3486
3487                 if (found) {
3488                         /* Replace key */
3489                         obj = ucl_object_ref (found);
3490                         ucl_object_delete_keyl (comments, (const char *)&from,
3491                                         sizeof (void *));
3492                         ucl_object_insert_key (comments, obj, (const char *)&to,
3493                                         sizeof (void *), true);
3494
3495                         return true;
3496                 }
3497         }
3498
3499         return false;
3500 }
3501
3502 void
3503 ucl_comments_add (ucl_object_t *comments, const ucl_object_t *obj,
3504                 const char *comment)
3505 {
3506         if (comments && obj && comment) {
3507                 ucl_object_insert_key (comments, ucl_object_fromstring (comment),
3508                                 (const char *)&obj, sizeof (void *), true);
3509         }
3510 }