contrib/libucl/src/ucl_util.c

   1 /* Copyright (c) 2013, Vsevolod Stakhov
   2  * Copyright (c) 2015 Allan Jude <allanjude@freebsd.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *       * Redistributions of source code must retain the above copyright
   8  *         notice, this list of conditions and the following disclaimer.
   9  *       * Redistributions in binary form must reproduce the above copyright
  10  *         notice, this list of conditions and the following disclaimer in the
  11  *         documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  15  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  16  * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
  17  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  18  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  19  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  20  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  22  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  23  */
  24
  25 #include "ucl.h"
  26 #include "ucl_internal.h"
  27 #include "ucl_chartable.h"
  28 #include "kvec.h"
  29 #include <stdarg.h>
  30
  31 #ifndef _WIN32
  32 #include <glob.h>
  33 #endif
  34
  35 #ifdef HAVE_LIBGEN_H
  36 #include <libgen.h> /* For dirname */
  37 #endif
  38
  39 typedef kvec_t(ucl_object_t *) ucl_array_t;
  40
  41 #define UCL_ARRAY_GET(ar, obj) ucl_array_t *ar = \
  42         (ucl_array_t *)((obj) != NULL ? (obj)->value.av : NULL)
  43
  44 #ifdef HAVE_OPENSSL
  45 #include <openssl/err.h>
  46 #include <openssl/sha.h>
  47 #include <openssl/rsa.h>
  48 #include <openssl/ssl.h>
  49 #include <openssl/evp.h>
  50 #endif
  51
  52 #ifdef CURL_FOUND
  53 #include <curl/curl.h>
  54 #endif
  55 #ifdef HAVE_FETCH_H
  56 #include <fetch.h>
  57 #endif
  58
  59 #ifdef _WIN32
  60 #include <windows.h>
  61
  62 #ifndef PROT_READ
  63 #define PROT_READ       1
  64 #endif
  65 #ifndef PROT_WRITE
  66 #define PROT_WRITE      2
  67 #endif
  68 #ifndef PROT_READWRITE
  69 #define PROT_READWRITE  3
  70 #endif
  71 #ifndef MAP_SHARED
  72 #define MAP_SHARED      1
  73 #endif
  74 #ifndef MAP_PRIVATE
  75 #define MAP_PRIVATE     2
  76 #endif
  77 #ifndef MAP_FAILED
  78 #define MAP_FAILED      ((void *) -1)
  79 #endif
  80
  81 #ifdef _WIN32
  82 #include <limits.h>
  83 #define NBBY CHAR_BIT
  84 #endif
  85
  86 static void *ucl_mmap(char *addr, size_t length, int prot, int access, int fd, off_t offset)
  87 {
  88         void *map = NULL;
  89         HANDLE handle = INVALID_HANDLE_VALUE;
  90
  91         switch (prot) {
  92         default:
  93         case PROT_READ:
  94                 {
  95                         handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READONLY, 0, length, 0);
  96                         if (!handle) break;
  97                         map = (void *) MapViewOfFile(handle, FILE_MAP_READ, 0, 0, length);
  98                         CloseHandle(handle);
  99                         break;
 100                 }
 101         case PROT_WRITE:
 102                 {
 103                         handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READWRITE, 0, length, 0);
 104                         if (!handle) break;
 105                         map = (void *) MapViewOfFile(handle, FILE_MAP_WRITE, 0, 0, length);
 106                         CloseHandle(handle);
 107                         break;
 108                 }
 109         case PROT_READWRITE:
 110                 {
 111                         handle = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, PAGE_READWRITE, 0, length, 0);
 112                         if (!handle) break;
 113                         map = (void *) MapViewOfFile(handle, FILE_MAP_ALL_ACCESS, 0, 0, length);
 114                         CloseHandle(handle);
 115                         break;
 116                 }
 117         }
 118         if (map == (void *) NULL) {
 119                 return (void *) MAP_FAILED;
 120         }
 121         return (void *) ((char *) map + offset);
 122 }
 123
 124 static int ucl_munmap(void *map,size_t length)
 125 {
 126         if (!UnmapViewOfFile(map)) {
 127                 return(-1);
 128         }
 129         return(0);
 130 }
 131
 132 static char* ucl_realpath(const char *path, char *resolved_path) {
 133     char *p;
 134     char tmp[MAX_PATH + 1];
 135     strncpy(tmp, path, sizeof(tmp)-1);
 136     p = tmp;
 137     while(*p) {
 138         if (*p == '/') *p = '\\';
 139         p++;
 140     }
 141     return _fullpath(resolved_path, tmp, MAX_PATH);
 142 }
 143 #else
 144 #define ucl_mmap mmap
 145 #define ucl_munmap munmap
 146 #define ucl_realpath realpath
 147 #endif
 148
 149 typedef void (*ucl_object_dtor) (ucl_object_t *obj);
 150 static void ucl_object_free_internal (ucl_object_t *obj, bool allow_rec,
 151                 ucl_object_dtor dtor);
 152 static void ucl_object_dtor_unref (ucl_object_t *obj);
 153
 154 static void
 155 ucl_object_dtor_free (ucl_object_t *obj)
 156 {
 157         if (obj->trash_stack[UCL_TRASH_KEY] != NULL) {
 158                 UCL_FREE (obj->hh.keylen, obj->trash_stack[UCL_TRASH_KEY]);
 159         }
 160         if (obj->trash_stack[UCL_TRASH_VALUE] != NULL) {
 161                 UCL_FREE (obj->len, obj->trash_stack[UCL_TRASH_VALUE]);
 162         }
 163         /* Do not free ephemeral objects */
 164         if ((obj->flags & UCL_OBJECT_EPHEMERAL) == 0) {
 165                 if (obj->type != UCL_USERDATA) {
 166                         UCL_FREE (sizeof (ucl_object_t), obj);
 167                 }
 168                 else {
 169                         struct ucl_object_userdata *ud = (struct ucl_object_userdata *)obj;
 170                         if (ud->dtor) {
 171                                 ud->dtor (obj->value.ud);
 172                         }
 173                         UCL_FREE (sizeof (*ud), obj);
 174                 }
 175         }
 176 }
 177
 178 /*
 179  * This is a helper function that performs exactly the same as
 180  * `ucl_object_unref` but it doesn't iterate over elements allowing
 181  * to use it for individual elements of arrays and multiple values
 182  */
 183 static void
 184 ucl_object_dtor_unref_single (ucl_object_t *obj)
 185 {
 186         if (obj != NULL) {
 187 #ifdef HAVE_ATOMIC_BUILTINS
 188                 unsigned int rc = __sync_sub_and_fetch (&obj->ref, 1);
 189                 if (rc == 0) {
 190 #else
 191                 if (--obj->ref == 0) {
 192 #endif
 193                         ucl_object_free_internal (obj, false, ucl_object_dtor_unref);
 194                 }
 195         }
 196 }
 197
 198 static void
 199 ucl_object_dtor_unref (ucl_object_t *obj)
 200 {
 201         if (obj->ref == 0) {
 202                 ucl_object_dtor_free (obj);
 203         }
 204         else {
 205                 /* This may cause dtor unref being called one more time */
 206                 ucl_object_dtor_unref_single (obj);
 207         }
 208 }
 209
 210 static void
 211 ucl_object_free_internal (ucl_object_t *obj, bool allow_rec, ucl_object_dtor dtor)
 212 {
 213         ucl_object_t *tmp, *sub;
 214
 215         while (obj != NULL) {
 216                 if (obj->type == UCL_ARRAY) {
 217                         UCL_ARRAY_GET (vec, obj);
 218                         unsigned int i;
 219
 220                         if (vec != NULL) {
 221                                 for (i = 0; i < vec->n; i ++) {
 222                                         sub = kv_A (*vec, i);
 223                                         if (sub != NULL) {
 224                                                 tmp = sub;
 225                                                 while (sub) {
 226                                                         tmp = sub->next;
 227                                                         dtor (sub);
 228                                                         sub = tmp;
 229                                                 }
 230                                         }
 231                                 }
 232                                 kv_destroy (*vec);
 233                                 UCL_FREE (sizeof (*vec), vec);
 234                         }
 235                         obj->value.av = NULL;
 236                 }
 237                 else if (obj->type == UCL_OBJECT) {
 238                         if (obj->value.ov != NULL) {
 239                                 ucl_hash_destroy (obj->value.ov, (ucl_hash_free_func *)dtor);
 240                         }
 241                         obj->value.ov = NULL;
 242                 }
 243                 tmp = obj->next;
 244                 dtor (obj);
 245                 obj = tmp;
 246
 247                 if (!allow_rec) {
 248                         break;
 249                 }
 250         }
 251 }
 252
 253 void
 254 ucl_object_free (ucl_object_t *obj)
 255 {
 256         ucl_object_free_internal (obj, true, ucl_object_dtor_free);
 257 }
 258
 259 size_t
 260 ucl_unescape_json_string (char *str, size_t len)
 261 {
 262         char *t = str, *h = str;
 263         int i, uval;
 264
 265         if (len <= 1) {
 266                 return len;
 267         }
 268         /* t is target (tortoise), h is source (hare) */
 269
 270         while (len) {
 271                 if (*h == '\\') {
 272                         h ++;
 273
 274                         if (len == 1) {
 275                                 /*
 276                                  * If \ is last, then do not try to go further
 277                                  * Issue: #74
 278                                  */
 279                                 len --;
 280                                 *t++ = '\\';
 281                                 continue;
 282                         }
 283
 284                         switch (*h) {
 285                         case 'n':
 286                                 *t++ = '\n';
 287                                 break;
 288                         case 'r':
 289                                 *t++ = '\r';
 290                                 break;
 291                         case 'b':
 292                                 *t++ = '\b';
 293                                 break;
 294                         case 't':
 295                                 *t++ = '\t';
 296                                 break;
 297                         case 'f':
 298                                 *t++ = '\f';
 299                                 break;
 300                         case '\\':
 301                                 *t++ = '\\';
 302                                 break;
 303                         case '"':
 304                                 *t++ = '"';
 305                                 break;
 306                         case 'u':
 307                                 /* Unicode escape */
 308                                 uval = 0;
 309                                 if (len > 3) {
 310                                         for (i = 0; i < 4; i++) {
 311                                                 uval <<= 4;
 312                                                 if (isdigit (h[i])) {
 313                                                         uval += h[i] - '0';
 314                                                 }
 315                                                 else if (h[i] >= 'a' && h[i] <= 'f') {
 316                                                         uval += h[i] - 'a' + 10;
 317                                                 }
 318                                                 else if (h[i] >= 'A' && h[i] <= 'F') {
 319                                                         uval += h[i] - 'A' + 10;
 320                                                 }
 321                                                 else {
 322                                                         break;
 323                                                 }
 324                                         }
 325                                         h += 3;
 326                                         len -= 3;
 327                                         /* Encode */
 328                                         if(uval < 0x80) {
 329                                                 t[0] = (char)uval;
 330                                                 t ++;
 331                                         }
 332                                         else if(uval < 0x800) {
 333                                                 t[0] = 0xC0 + ((uval & 0x7C0) >> 6);
 334                                                 t[1] = 0x80 + ((uval & 0x03F));
 335                                                 t += 2;
 336                                         }
 337                                         else if(uval < 0x10000) {
 338                                                 t[0] = 0xE0 + ((uval & 0xF000) >> 12);
 339                                                 t[1] = 0x80 + ((uval & 0x0FC0) >> 6);
 340                                                 t[2] = 0x80 + ((uval & 0x003F));
 341                                                 t += 3;
 342                                         }
 343                                         else if(uval <= 0x10FFFF) {
 344                                                 t[0] = 0xF0 + ((uval & 0x1C0000) >> 18);
 345                                                 t[1] = 0x80 + ((uval & 0x03F000) >> 12);
 346                                                 t[2] = 0x80 + ((uval & 0x000FC0) >> 6);
 347                                                 t[3] = 0x80 + ((uval & 0x00003F));
 348                                                 t += 4;
 349                                         }
 350                                         else {
 351                                                 *t++ = '?';
 352                                         }
 353                                 }
 354                                 else {
 355                                         *t++ = 'u';
 356                                 }
 357                                 break;
 358                         default:
 359                                 *t++ = *h;
 360                                 break;
 361                         }
 362                         h ++;
 363                         len --;
 364                 }
 365                 else {
 366                         *t++ = *h++;
 367                 }
 368
 369                 if (len > 0) {
 370                         len --;
 371                 }
 372         }
 373         *t = '\0';
 374
 375         return (t - str);
 376 }
 377
 378 char *
 379 ucl_copy_key_trash (const ucl_object_t *obj)
 380 {
 381         ucl_object_t *deconst;
 382
 383         if (obj == NULL) {
 384                 return NULL;
 385         }
 386         if (obj->trash_stack[UCL_TRASH_KEY] == NULL && obj->key != NULL) {
 387                 deconst = __DECONST (ucl_object_t *, obj);
 388                 deconst->trash_stack[UCL_TRASH_KEY] = malloc (obj->keylen + 1);
 389                 if (deconst->trash_stack[UCL_TRASH_KEY] != NULL) {
 390                         memcpy (deconst->trash_stack[UCL_TRASH_KEY], obj->key, obj->keylen);
 391                         deconst->trash_stack[UCL_TRASH_KEY][obj->keylen] = '\0';
 392                 }
 393                 deconst->key = obj->trash_stack[UCL_TRASH_KEY];
 394                 deconst->flags |= UCL_OBJECT_ALLOCATED_KEY;
 395         }
 396
 397         return obj->trash_stack[UCL_TRASH_KEY];
 398 }
 399
 400 char *
 401 ucl_copy_value_trash (const ucl_object_t *obj)
 402 {
 403         ucl_object_t *deconst;
 404
 405         if (obj == NULL) {
 406                 return NULL;
 407         }
 408         if (obj->trash_stack[UCL_TRASH_VALUE] == NULL) {
 409                 deconst = __DECONST (ucl_object_t *, obj);
 410                 if (obj->type == UCL_STRING) {
 411
 412                         /* Special case for strings */
 413                         if (obj->flags & UCL_OBJECT_BINARY) {
 414                                 deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len);
 415                                 if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) {
 416                                         memcpy (deconst->trash_stack[UCL_TRASH_VALUE],
 417                                                         obj->value.sv,
 418                                                         obj->len);
 419                                         deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE];
 420                                 }
 421                         }
 422                         else {
 423                                 deconst->trash_stack[UCL_TRASH_VALUE] = malloc (obj->len + 1);
 424                                 if (deconst->trash_stack[UCL_TRASH_VALUE] != NULL) {
 425                                         memcpy (deconst->trash_stack[UCL_TRASH_VALUE],
 426                                                         obj->value.sv,
 427                                                         obj->len);
 428                                         deconst->trash_stack[UCL_TRASH_VALUE][obj->len] = '\0';
 429                                         deconst->value.sv = obj->trash_stack[UCL_TRASH_VALUE];
 430                                 }
 431                         }
 432                 }
 433                 else {
 434                         /* Just emit value in json notation */
 435                         deconst->trash_stack[UCL_TRASH_VALUE] = ucl_object_emit_single_json (obj);
 436                         deconst->len = strlen (obj->trash_stack[UCL_TRASH_VALUE]);
 437                 }
 438                 deconst->flags |= UCL_OBJECT_ALLOCATED_VALUE;
 439         }
 440
 441         return obj->trash_stack[UCL_TRASH_VALUE];
 442 }
 443
 444 ucl_object_t*
 445 ucl_parser_get_object (struct ucl_parser *parser)
 446 {
 447         if (parser->state != UCL_STATE_ERROR && parser->top_obj != NULL) {
 448                 return ucl_object_ref (parser->top_obj);
 449         }
 450
 451         return NULL;
 452 }
 453
 454 void
 455 ucl_parser_free (struct ucl_parser *parser)
 456 {
 457         struct ucl_stack *stack, *stmp;
 458         struct ucl_macro *macro, *mtmp;
 459         struct ucl_chunk *chunk, *ctmp;
 460         struct ucl_pubkey *key, *ktmp;
 461         struct ucl_variable *var, *vtmp;
 462         ucl_object_t *tr, *trtmp;
 463
 464         if (parser == NULL) {
 465                 return;
 466         }
 467
 468         if (parser->top_obj != NULL) {
 469                 ucl_object_unref (parser->top_obj);
 470         }
 471
 472         if (parser->includepaths != NULL) {
 473                 ucl_object_unref (parser->includepaths);
 474         }
 475
 476         LL_FOREACH_SAFE (parser->stack, stack, stmp) {
 477                 free (stack);
 478         }
 479         HASH_ITER (hh, parser->macroes, macro, mtmp) {
 480                 free (macro->name);
 481                 HASH_DEL (parser->macroes, macro);
 482                 UCL_FREE (sizeof (struct ucl_macro), macro);
 483         }
 484         LL_FOREACH_SAFE (parser->chunks, chunk, ctmp) {
 485                 UCL_FREE (sizeof (struct ucl_chunk), chunk);
 486         }
 487         LL_FOREACH_SAFE (parser->keys, key, ktmp) {
 488                 UCL_FREE (sizeof (struct ucl_pubkey), key);
 489         }
 490         LL_FOREACH_SAFE (parser->variables, var, vtmp) {
 491                 free (var->value);
 492                 free (var->var);
 493                 UCL_FREE (sizeof (struct ucl_variable), var);
 494         }
 495         LL_FOREACH_SAFE (parser->trash_objs, tr, trtmp) {
 496                 ucl_object_free_internal (tr, false, ucl_object_dtor_free);
 497         }
 498
 499         if (parser->err != NULL) {
 500                 utstring_free (parser->err);
 501         }
 502
 503         if (parser->cur_file) {
 504                 free (parser->cur_file);
 505         }
 506
 507         UCL_FREE (sizeof (struct ucl_parser), parser);
 508 }
 509
 510 const char *
 511 ucl_parser_get_error(struct ucl_parser *parser)
 512 {
 513         if (parser == NULL) {
 514                 return NULL;
 515         }
 516
 517         if (parser->err == NULL) {
 518                 return NULL;
 519         }
 520
 521         return utstring_body (parser->err);
 522 }
 523
 524 int
 525 ucl_parser_get_error_code(struct ucl_parser *parser)
 526 {
 527         if (parser == NULL) {
 528                 return 0;
 529         }
 530
 531         return parser->err_code;
 532 }
 533
 534 unsigned
 535 ucl_parser_get_column(struct ucl_parser *parser)
 536 {
 537         if (parser == NULL || parser->chunks == NULL) {
 538                 return 0;
 539         }
 540
 541         return parser->chunks->column;
 542 }
 543
 544 unsigned
 545 ucl_parser_get_linenum(struct ucl_parser *parser)
 546 {
 547         if (parser == NULL || parser->chunks == NULL) {
 548                 return 0;
 549         }
 550
 551         return parser->chunks->line;
 552 }
 553
 554 void
 555 ucl_parser_clear_error(struct ucl_parser *parser)
 556 {
 557         if (parser != NULL && parser->err != NULL) {
 558                 utstring_free(parser->err);
 559                 parser->err = NULL;
 560                 parser->err_code = 0;
 561         }
 562 }
 563
 564 bool
 565 ucl_pubkey_add (struct ucl_parser *parser, const unsigned char *key, size_t len)
 566 {
 567 #ifndef HAVE_OPENSSL
 568         ucl_create_err (&parser->err, "cannot check signatures without openssl");
 569         return false;
 570 #else
 571 # if (OPENSSL_VERSION_NUMBER < 0x10000000L)
 572         ucl_create_err (&parser->err, "cannot check signatures, openssl version is unsupported");
 573         return EXIT_FAILURE;
 574 # else
 575         struct ucl_pubkey *nkey;
 576         BIO *mem;
 577
 578         mem = BIO_new_mem_buf ((void *)key, len);
 579         nkey = UCL_ALLOC (sizeof (struct ucl_pubkey));
 580         if (nkey == NULL) {
 581                 ucl_create_err (&parser->err, "cannot allocate memory for key");
 582                 return false;
 583         }
 584         nkey->key = PEM_read_bio_PUBKEY (mem, &nkey->key, NULL, NULL);
 585         BIO_free (mem);
 586         if (nkey->key == NULL) {
 587                 UCL_FREE (sizeof (struct ucl_pubkey), nkey);
 588                 ucl_create_err (&parser->err, "%s",
 589                                 ERR_error_string (ERR_get_error (), NULL));
 590                 return false;
 591         }
 592         LL_PREPEND (parser->keys, nkey);
 593 # endif
 594 #endif
 595         return true;
 596 }
 597
 598 #ifdef CURL_FOUND
 599 struct ucl_curl_cbdata {
 600         unsigned char *buf;
 601         size_t buflen;
 602 };
 603
 604 static size_t
 605 ucl_curl_write_callback (void* contents, size_t size, size_t nmemb, void* ud)
 606 {
 607         struct ucl_curl_cbdata *cbdata = ud;
 608         size_t realsize = size * nmemb;
 609
 610         cbdata->buf = realloc (cbdata->buf, cbdata->buflen + realsize + 1);
 611         if (cbdata->buf == NULL) {
 612                 return 0;
 613         }
 614
 615         memcpy (&(cbdata->buf[cbdata->buflen]), contents, realsize);
 616         cbdata->buflen += realsize;
 617         cbdata->buf[cbdata->buflen] = 0;
 618
 619         return realsize;
 620 }
 621 #endif
 622
 623 /**
 624  * Fetch a url and save results to the memory buffer
 625  * @param url url to fetch
 626  * @param len length of url
 627  * @param buf target buffer
 628  * @param buflen target length
 629  * @return
 630  */
 631 static bool
 632 ucl_fetch_url (const unsigned char *url, unsigned char **buf, size_t *buflen,
 633                 UT_string **err, bool must_exist)
 634 {
 635
 636 #ifdef HAVE_FETCH_H
 637         struct url *fetch_url;
 638         struct url_stat us;
 639         FILE *in;
 640
 641         fetch_url = fetchParseURL (url);
 642         if (fetch_url == NULL) {
 643                 ucl_create_err (err, "invalid URL %s: %s",
 644                                 url, strerror (errno));
 645                 return false;
 646         }
 647         if ((in = fetchXGet (fetch_url, &us, "")) == NULL) {
 648                 if (!must_exist) {
 649                         ucl_create_err (err, "cannot fetch URL %s: %s",
 650                                 url, strerror (errno));
 651                 }
 652                 fetchFreeURL (fetch_url);
 653                 return false;
 654         }
 655
 656         *buflen = us.size;
 657         *buf = malloc (*buflen);
 658         if (*buf == NULL) {
 659                 ucl_create_err (err, "cannot allocate buffer for URL %s: %s",
 660                                 url, strerror (errno));
 661                 fclose (in);
 662                 fetchFreeURL (fetch_url);
 663                 return false;
 664         }
 665
 666         if (fread (*buf, *buflen, 1, in) != 1) {
 667                 ucl_create_err (err, "cannot read URL %s: %s",
 668                                 url, strerror (errno));
 669                 fclose (in);
 670                 fetchFreeURL (fetch_url);
 671                 return false;
 672         }
 673
 674         fetchFreeURL (fetch_url);
 675         return true;
 676 #elif defined(CURL_FOUND)
 677         CURL *curl;
 678         int r;
 679         struct ucl_curl_cbdata cbdata;
 680
 681         curl = curl_easy_init ();
 682         if (curl == NULL) {
 683                 ucl_create_err (err, "CURL interface is broken");
 684                 return false;
 685         }
 686         if ((r = curl_easy_setopt (curl, CURLOPT_URL, url)) != CURLE_OK) {
 687                 ucl_create_err (err, "invalid URL %s: %s",
 688                                 url, curl_easy_strerror (r));
 689                 curl_easy_cleanup (curl);
 690                 return false;
 691         }
 692         curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, ucl_curl_write_callback);
 693         cbdata.buf = *buf;
 694         cbdata.buflen = *buflen;
 695         curl_easy_setopt (curl, CURLOPT_WRITEDATA, &cbdata);
 696
 697         if ((r = curl_easy_perform (curl)) != CURLE_OK) {
 698                 if (!must_exist) {
 699                         ucl_create_err (err, "error fetching URL %s: %s",
 700                                 url, curl_easy_strerror (r));
 701                 }
 702                 curl_easy_cleanup (curl);
 703                 if (cbdata.buf) {
 704                         free (cbdata.buf);
 705                 }
 706                 return false;
 707         }
 708         *buf = cbdata.buf;
 709         *buflen = cbdata.buflen;
 710
 711         return true;
 712 #else
 713         ucl_create_err (err, "URL support is disabled");
 714         return false;
 715 #endif
 716 }
 717
 718 /**
 719  * Fetch a file and save results to the memory buffer
 720  * @param filename filename to fetch
 721  * @param len length of filename
 722  * @param buf target buffer
 723  * @param buflen target length
 724  * @return
 725  */
 726 static bool
 727 ucl_fetch_file (const unsigned char *filename, unsigned char **buf, size_t *buflen,
 728                 UT_string **err, bool must_exist)
 729 {
 730         int fd;
 731         struct stat st;
 732
 733         if (stat (filename, &st) == -1 || !S_ISREG (st.st_mode)) {
 734                 if (must_exist) {
 735                         ucl_create_err (err, "cannot stat file %s: %s",
 736                                         filename, strerror (errno));
 737                 }
 738                 return false;
 739         }
 740         if (st.st_size == 0) {
 741                 /* Do not map empty files */
 742                 *buf = "";
 743                 *buflen = 0;
 744         }
 745         else {
 746                 if ((fd = open (filename, O_RDONLY)) == -1) {
 747                         ucl_create_err (err, "cannot open file %s: %s",
 748                                         filename, strerror (errno));
 749                         return false;
 750                 }
 751                 if ((*buf = ucl_mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
 752                         close (fd);
 753                         ucl_create_err (err, "cannot mmap file %s: %s",
 754                                         filename, strerror (errno));
 755                         return false;
 756                 }
 757                 *buflen = st.st_size;
 758                 close (fd);
 759         }
 760
 761         return true;
 762 }
 763
 764
 765 #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
 766 static inline bool
 767 ucl_sig_check (const unsigned char *data, size_t datalen,
 768                 const unsigned char *sig, size_t siglen, struct ucl_parser *parser)
 769 {
 770         struct ucl_pubkey *key;
 771         char dig[EVP_MAX_MD_SIZE];
 772         unsigned int diglen;
 773         EVP_PKEY_CTX *key_ctx;
 774         EVP_MD_CTX *sign_ctx = NULL;
 775
 776         sign_ctx = EVP_MD_CTX_create ();
 777
 778         LL_FOREACH (parser->keys, key) {
 779                 key_ctx = EVP_PKEY_CTX_new (key->key, NULL);
 780                 if (key_ctx != NULL) {
 781                         if (EVP_PKEY_verify_init (key_ctx) <= 0) {
 782                                 EVP_PKEY_CTX_free (key_ctx);
 783                                 continue;
 784                         }
 785                         if (EVP_PKEY_CTX_set_rsa_padding (key_ctx, RSA_PKCS1_PADDING) <= 0) {
 786                                 EVP_PKEY_CTX_free (key_ctx);
 787                                 continue;
 788                         }
 789                         if (EVP_PKEY_CTX_set_signature_md (key_ctx, EVP_sha256 ()) <= 0) {
 790                                 EVP_PKEY_CTX_free (key_ctx);
 791                                 continue;
 792                         }
 793                         EVP_DigestInit (sign_ctx, EVP_sha256 ());
 794                         EVP_DigestUpdate (sign_ctx, data, datalen);
 795                         EVP_DigestFinal (sign_ctx, dig, &diglen);
 796
 797                         if (EVP_PKEY_verify (key_ctx, sig, siglen, dig, diglen) == 1) {
 798                                 EVP_MD_CTX_destroy (sign_ctx);
 799                                 EVP_PKEY_CTX_free (key_ctx);
 800                                 return true;
 801                         }
 802
 803                         EVP_PKEY_CTX_free (key_ctx);
 804                 }
 805         }
 806
 807         EVP_MD_CTX_destroy (sign_ctx);
 808
 809         return false;
 810 }
 811 #endif
 812
 813 struct ucl_include_params {
 814         bool check_signature;
 815         bool must_exist;
 816         bool use_glob;
 817         bool use_prefix;
 818         bool soft_fail;
 819         bool allow_glob;
 820         unsigned priority;
 821         enum ucl_duplicate_strategy strat;
 822         enum ucl_parse_type parse_type;
 823         const char *prefix;
 824         const char *target;
 825 };
 826
 827 /**
 828  * Include an url to configuration
 829  * @param data
 830  * @param len
 831  * @param parser
 832  * @param err
 833  * @return
 834  */
 835 static bool
 836 ucl_include_url (const unsigned char *data, size_t len,
 837                 struct ucl_parser *parser,
 838                 struct ucl_include_params *params)
 839 {
 840
 841         bool res;
 842         unsigned char *buf = NULL;
 843         size_t buflen = 0;
 844         struct ucl_chunk *chunk;
 845         char urlbuf[PATH_MAX];
 846         int prev_state;
 847
 848         snprintf (urlbuf, sizeof (urlbuf), "%.*s", (int)len, data);
 849
 850         if (!ucl_fetch_url (urlbuf, &buf, &buflen, &parser->err, params->must_exist)) {
 851                 return (!params->must_exist || false);
 852         }
 853
 854         if (params->check_signature) {
 855 #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
 856                 unsigned char *sigbuf = NULL;
 857                 size_t siglen = 0;
 858                 /* We need to check signature first */
 859                 snprintf (urlbuf, sizeof (urlbuf), "%.*s.sig", (int)len, data);
 860                 if (!ucl_fetch_url (urlbuf, &sigbuf, &siglen, &parser->err, true)) {
 861                         return false;
 862                 }
 863                 if (!ucl_sig_check (buf, buflen, sigbuf, siglen, parser)) {
 864                         ucl_create_err (&parser->err, "cannot verify url %s: %s",
 865                                                         urlbuf,
 866                                                         ERR_error_string (ERR_get_error (), NULL));
 867                         if (siglen > 0) {
 868                                 ucl_munmap (sigbuf, siglen);
 869                         }
 870                         return false;
 871                 }
 872                 if (siglen > 0) {
 873                         ucl_munmap (sigbuf, siglen);
 874                 }
 875 #endif
 876         }
 877
 878         prev_state = parser->state;
 879         parser->state = UCL_STATE_INIT;
 880
 881         res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority,
 882                         params->strat, params->parse_type);
 883         if (res == true) {
 884                 /* Remove chunk from the stack */
 885                 chunk = parser->chunks;
 886                 if (chunk != NULL) {
 887                         parser->chunks = chunk->next;
 888                         UCL_FREE (sizeof (struct ucl_chunk), chunk);
 889                 }
 890         }
 891
 892         parser->state = prev_state;
 893         free (buf);
 894
 895         return res;
 896 }
 897
 898 /**
 899  * Include a single file to the parser
 900  * @param data
 901  * @param len
 902  * @param parser
 903  * @param check_signature
 904  * @param must_exist
 905  * @param allow_glob
 906  * @param priority
 907  * @return
 908  */
 909 static bool
 910 ucl_include_file_single (const unsigned char *data, size_t len,
 911                 struct ucl_parser *parser, struct ucl_include_params *params)
 912 {
 913         bool res;
 914         struct ucl_chunk *chunk;
 915         unsigned char *buf = NULL;
 916         char *old_curfile, *ext;
 917         size_t buflen = 0;
 918         char filebuf[PATH_MAX], realbuf[PATH_MAX];
 919         int prev_state;
 920         struct ucl_variable *cur_var, *tmp_var, *old_curdir = NULL,
 921                         *old_filename = NULL;
 922         ucl_object_t *nest_obj = NULL, *old_obj = NULL, *new_obj = NULL;
 923         ucl_hash_t *container = NULL;
 924         struct ucl_stack *st = NULL;
 925
 926         snprintf (filebuf, sizeof (filebuf), "%.*s", (int)len, data);
 927         if (ucl_realpath (filebuf, realbuf) == NULL) {
 928                 if (params->soft_fail) {
 929                         return false;
 930                 }
 931                 if (!params->must_exist) {
 932                         return true;
 933                 }
 934                 ucl_create_err (&parser->err, "cannot open file %s: %s",
 935                                                                         filebuf,
 936                                                                         strerror (errno));
 937                 return false;
 938         }
 939
 940         if (parser->cur_file && strcmp (realbuf, parser->cur_file) == 0) {
 941                 /* We are likely including the file itself */
 942                 if (params->soft_fail) {
 943                         return false;
 944                 }
 945
 946                 ucl_create_err (&parser->err, "trying to include the file %s from itself",
 947                                 realbuf);
 948                 return false;
 949         }
 950
 951         if (!ucl_fetch_file (realbuf, &buf, &buflen, &parser->err, params->must_exist)) {
 952                 if (params->soft_fail) {
 953                         return false;
 954                 }
 955                 return (!params->must_exist || false);
 956         }
 957
 958         if (params->check_signature) {
 959 #if (defined(HAVE_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10000000L)
 960                 unsigned char *sigbuf = NULL;
 961                 size_t siglen = 0;
 962                 /* We need to check signature first */
 963                 snprintf (filebuf, sizeof (filebuf), "%s.sig", realbuf);
 964                 if (!ucl_fetch_file (filebuf, &sigbuf, &siglen, &parser->err, true)) {
 965                         return false;
 966                 }
 967                 if (!ucl_sig_check (buf, buflen, sigbuf, siglen, parser)) {
 968                         ucl_create_err (&parser->err, "cannot verify file %s: %s",
 969                                                         filebuf,
 970                                                         ERR_error_string (ERR_get_error (), NULL));
 971                         if (siglen > 0) {
 972                                 ucl_munmap (sigbuf, siglen);
 973                         }
 974                         return false;
 975                 }
 976                 if (siglen > 0) {
 977                         ucl_munmap (sigbuf, siglen);
 978                 }
 979 #endif
 980         }
 981
 982         old_curfile = parser->cur_file;
 983         parser->cur_file = strdup (realbuf);
 984
 985         /* Store old file vars */
 986         DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) {
 987                 if (strcmp (cur_var->var, "CURDIR") == 0) {
 988                         old_curdir = cur_var;
 989                         DL_DELETE (parser->variables, cur_var);
 990                 }
 991                 else if (strcmp (cur_var->var, "FILENAME") == 0) {
 992                         old_filename = cur_var;
 993                         DL_DELETE (parser->variables, cur_var);
 994                 }
 995         }
 996
 997         ucl_parser_set_filevars (parser, realbuf, false);
 998
 999         prev_state = parser->state;
1000         parser->state = UCL_STATE_INIT;
1001
1002         if (params->use_prefix && params->prefix == NULL) {
1003                 /* Auto generate a key name based on the included filename */
1004                 params->prefix = basename (realbuf);
1005                 ext = strrchr (params->prefix, '.');
1006                 if (ext != NULL && (strcmp (ext, ".conf") == 0 || strcmp (ext, ".ucl") == 0)) {
1007                         /* Strip off .conf or .ucl */
1008                         *ext = '\0';
1009                 }
1010         }
1011         if (params->prefix != NULL) {
1012                 /* This is a prefixed include */
1013                 container = parser->stack->obj->value.ov;
1014
1015                 old_obj = __DECONST (ucl_object_t *, ucl_hash_search (container,
1016                                 params->prefix, strlen (params->prefix)));
1017
1018                 if (strcasecmp (params->target, "array") == 0 && old_obj == NULL) {
1019                         /* Create an array with key: prefix */
1020                         old_obj = ucl_object_new_full (UCL_ARRAY, params->priority);
1021                         old_obj->key = params->prefix;
1022                         old_obj->keylen = strlen (params->prefix);
1023                         ucl_copy_key_trash(old_obj);
1024                         old_obj->prev = old_obj;
1025                         old_obj->next = NULL;
1026
1027                         container = ucl_hash_insert_object (container, old_obj,
1028                                         parser->flags & UCL_PARSER_KEY_LOWERCASE);
1029                         parser->stack->obj->len ++;
1030
1031                         nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
1032                         nest_obj->prev = nest_obj;
1033                         nest_obj->next = NULL;
1034
1035                         ucl_array_append (old_obj, nest_obj);
1036                 }
1037                 else if (old_obj == NULL) {
1038                         /* Create an object with key: prefix */
1039                         nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
1040                         nest_obj->key = params->prefix;
1041                         nest_obj->keylen = strlen (params->prefix);
1042                         ucl_copy_key_trash(nest_obj);
1043                         nest_obj->prev = nest_obj;
1044                         nest_obj->next = NULL;
1045
1046                         container = ucl_hash_insert_object (container, nest_obj,
1047                                         parser->flags & UCL_PARSER_KEY_LOWERCASE);
1048                         parser->stack->obj->len ++;
1049                 }
1050                 else if (strcasecmp (params->target, "array") == 0 ||
1051                                 ucl_object_type(old_obj) == UCL_ARRAY) {
1052                         if (ucl_object_type(old_obj) == UCL_ARRAY) {
1053                                 /* Append to the existing array */
1054                                 nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
1055                                 nest_obj->prev = nest_obj;
1056                                 nest_obj->next = NULL;
1057
1058                                 ucl_array_append (old_obj, nest_obj);
1059                         }
1060                         else {
1061                                 /* Convert the object to an array */
1062                                 new_obj = ucl_object_typed_new (UCL_ARRAY);
1063                                 new_obj->key = old_obj->key;
1064                                 new_obj->keylen = old_obj->keylen;
1065                                 new_obj->flags |= UCL_OBJECT_MULTIVALUE;
1066                                 new_obj->prev = new_obj;
1067                                 new_obj->next = NULL;
1068
1069                                 nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority);
1070                                 nest_obj->prev = nest_obj;
1071                                 nest_obj->next = NULL;
1072
1073                                 ucl_array_append (new_obj, old_obj);
1074                                 ucl_array_append (new_obj, nest_obj);
1075                                 ucl_hash_replace (container, old_obj, new_obj);
1076                         }
1077                 }
1078                 else {
1079                         if (ucl_object_type (old_obj) == UCL_OBJECT) {
1080                                 /* Append to existing Object*/
1081                                 nest_obj = old_obj;
1082                         }
1083                         else {
1084                                 /* The key is not an object */
1085                                 ucl_create_err (&parser->err,
1086                                                 "Conflicting type for key: %s",
1087                                                 params->prefix);
1088                                 return false;
1089                         }
1090                 }
1091
1092                  /* Put all of the content of the include inside that object */
1093                 parser->stack->obj->value.ov = container;
1094
1095                 if (nest_obj != NULL) {
1096                         st = UCL_ALLOC (sizeof (struct ucl_stack));
1097                         if (st == NULL) {
1098                                 ucl_create_err (&parser->err, "cannot allocate memory for an object");
1099                                 ucl_object_unref (nest_obj);
1100                                 return NULL;
1101                         }
1102                         st->obj = nest_obj;
1103                         st->level = parser->stack->level;
1104                         LL_PREPEND (parser->stack, st);
1105                         parser->cur_obj = nest_obj;
1106                 }
1107         }
1108
1109         res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority,
1110                         params->strat, params->parse_type);
1111         if (!res && !params->must_exist) {
1112                 /* Free error */
1113                 utstring_free (parser->err);
1114                 parser->err = NULL;
1115                 parser->state = UCL_STATE_AFTER_VALUE;
1116         }
1117
1118         /* Stop nesting the include, take 1 level off the stack */
1119         if (params->prefix != NULL && nest_obj != NULL) {
1120                 parser->stack = st->next;
1121                 UCL_FREE (sizeof (struct ucl_stack), st);
1122         }
1123
1124         /* Remove chunk from the stack */
1125         chunk = parser->chunks;
1126         if (chunk != NULL) {
1127                 parser->chunks = chunk->next;
1128                 UCL_FREE (sizeof (struct ucl_chunk), chunk);
1129                 parser->recursion --;
1130         }
1131
1132         /* Restore old file vars */
1133         if (parser->cur_file) {
1134                 free (parser->cur_file);
1135         }
1136
1137         parser->cur_file = old_curfile;
1138         DL_FOREACH_SAFE (parser->variables, cur_var, tmp_var) {
1139                 if (strcmp (cur_var->var, "CURDIR") == 0 && old_curdir) {
1140                         DL_DELETE (parser->variables, cur_var);
1141                         free (cur_var->var);
1142                         free (cur_var->value);
1143                         UCL_FREE (sizeof (struct ucl_variable), cur_var);
1144                 }
1145                 else if (strcmp (cur_var->var, "FILENAME") == 0 && old_filename) {
1146                         DL_DELETE (parser->variables, cur_var);
1147                         free (cur_var->var);
1148                         free (cur_var->value);
1149                         UCL_FREE (sizeof (struct ucl_variable), cur_var);
1150                 }
1151         }
1152         if (old_filename) {
1153                 DL_APPEND (parser->variables, old_filename);
1154         }
1155         if (old_curdir) {
1156                 DL_APPEND (parser->variables, old_curdir);
1157         }
1158
1159         parser->state = prev_state;
1160
1161         if (buflen > 0) {
1162                 ucl_munmap (buf, buflen);
1163         }
1164
1165         return res;
1166 }
1167
1168 /**
1169  * Include a file to configuration
1170  * @param data
1171  * @param len
1172  * @param parser
1173  * @param err
1174  * @return
1175  */
1176 static bool
1177 ucl_include_file (const unsigned char *data, size_t len,
1178                 struct ucl_parser *parser, struct ucl_include_params *params)
1179 {
1180         const unsigned char *p = data, *end = data + len;
1181         bool need_glob = false;
1182         int cnt = 0;
1183         char glob_pattern[PATH_MAX];
1184         size_t i;
1185
1186 #ifndef _WIN32
1187         if (!params->allow_glob) {
1188                 return ucl_include_file_single (data, len, parser, params);
1189         }
1190         else {
1191                 /* Check for special symbols in a filename */
1192                 while (p != end) {
1193                         if (*p == '*' || *p == '?') {
1194                                 need_glob = true;
1195                                 break;
1196                         }
1197                         p ++;
1198                 }
1199                 if (need_glob) {
1200                         glob_t globbuf;
1201                         memset (&globbuf, 0, sizeof (globbuf));
1202                         ucl_strlcpy (glob_pattern, (const char *)data,
1203                                 (len + 1 < sizeof (glob_pattern) ? len + 1 : sizeof (glob_pattern)));
1204                         if (glob (glob_pattern, 0, NULL, &globbuf) != 0) {
1205                                 return (!params->must_exist || false);
1206                         }
1207                         for (i = 0; i < globbuf.gl_pathc; i ++) {
1208                                 if (!ucl_include_file_single ((unsigned char *)globbuf.gl_pathv[i],
1209                                                 strlen (globbuf.gl_pathv[i]), parser, params)) {
1210                                         if (params->soft_fail) {
1211                                                 continue;
1212                                         }
1213                                         globfree (&globbuf);
1214                                         return false;
1215                                 }
1216                                 cnt ++;
1217                         }
1218                         globfree (&globbuf);
1219
1220                         if (cnt == 0 && params->must_exist) {
1221                                 ucl_create_err (&parser->err, "cannot match any files for pattern %s",
1222                                         glob_pattern);
1223                                 return false;
1224                         }
1225                 }
1226                 else {
1227                         return ucl_include_file_single (data, len, parser, params);
1228                 }
1229         }
1230 #else
1231         /* Win32 compilers do not support globbing. Therefore, for Win32,
1232            treat allow_glob/need_glob as a NOOP and just return */
1233         return ucl_include_file_single (data, len, parser, params);
1234 #endif
1235
1236         return true;
1237 }
1238
1239 /**
1240  * Common function to handle .*include* macros
1241  * @param data
1242  * @param len
1243  * @param args
1244  * @param parser
1245  * @param default_try
1246  * @param default_sign
1247  * @return
1248  */
1249 static bool
1250 ucl_include_common (const unsigned char *data, size_t len,
1251                 const ucl_object_t *args, struct ucl_parser *parser,
1252                 bool default_try,
1253                 bool default_sign)
1254 {
1255         bool allow_url, search;
1256         const char *duplicate;
1257         const ucl_object_t *param;
1258         ucl_object_iter_t it = NULL, ip = NULL;
1259         char ipath[PATH_MAX];
1260         struct ucl_include_params params;
1261
1262         /* Default values */
1263         params.soft_fail = default_try;
1264         params.allow_glob = false;
1265         params.check_signature = default_sign;
1266         params.use_prefix = false;
1267         params.target = "object";
1268         params.prefix = NULL;
1269         params.priority = 0;
1270         params.parse_type = UCL_PARSE_UCL;
1271         params.strat = UCL_DUPLICATE_APPEND;
1272         params.must_exist = !default_try;
1273
1274         search = false;
1275
1276         /* Process arguments */
1277         if (args != NULL && args->type == UCL_OBJECT) {
1278                 while ((param = ucl_iterate_object (args, &it, true)) != NULL) {
1279                         if (param->type == UCL_BOOLEAN) {
1280                                 if (strncmp (param->key, "try", param->keylen) == 0) {
1281                                         params.must_exist = !ucl_object_toboolean (param);
1282                                 }
1283                                 else if (strncmp (param->key, "sign", param->keylen) == 0) {
1284                                         params.check_signature = ucl_object_toboolean (param);
1285                                 }
1286                                 else if (strncmp (param->key, "glob", param->keylen) == 0) {
1287                                         params.allow_glob = ucl_object_toboolean (param);
1288                                 }
1289                                 else if (strncmp (param->key, "url", param->keylen) == 0) {
1290                                         allow_url = ucl_object_toboolean (param);
1291                                 }
1292                                 else if (strncmp (param->key, "prefix", param->keylen) == 0) {
1293                                         params.use_prefix = ucl_object_toboolean (param);
1294                                 }
1295                         }
1296                         else if (param->type == UCL_STRING) {
1297                                 if (strncmp (param->key, "key", param->keylen) == 0) {
1298                                         params.prefix = ucl_object_tostring (param);
1299                                 }
1300                                 else if (strncmp (param->key, "target", param->keylen) == 0) {
1301                                         params.target = ucl_object_tostring (param);
1302                                 }
1303                                 else if (strncmp (param->key, "duplicate", param->keylen) == 0) {
1304                                         duplicate = ucl_object_tostring (param);
1305
1306                                         if (strcmp (duplicate, "append") == 0) {
1307                                                 params.strat = UCL_DUPLICATE_APPEND;
1308                                         }
1309                                         else if (strcmp (duplicate, "merge") == 0) {
1310                                                 params.strat = UCL_DUPLICATE_MERGE;
1311                                         }
1312                                         else if (strcmp (duplicate, "rewrite") == 0) {
1313                                                 params.strat = UCL_DUPLICATE_REWRITE;
1314                                         }
1315                                         else if (strcmp (duplicate, "error") == 0) {
1316                                                 params.strat = UCL_DUPLICATE_ERROR;
1317                                         }
1318                                 }
1319                         }
1320                         else if (param->type == UCL_ARRAY) {
1321                                 if (strncmp (param->key, "path", param->keylen) == 0) {
1322                                         ucl_set_include_path (parser, __DECONST(ucl_object_t *, param));
1323                                 }
1324                         }
1325                         else if (param->type == UCL_INT) {
1326                                 if (strncmp (param->key, "priority", param->keylen) == 0) {
1327                                         params.priority = ucl_object_toint (param);
1328                                 }
1329                         }
1330                 }
1331         }
1332
1333         if (parser->includepaths == NULL) {
1334                 if (allow_url && ucl_strnstr (data, "://", len) != NULL) {
1335                         /* Globbing is not used for URL's */
1336                         return ucl_include_url (data, len, parser, &params);
1337                 }
1338                 else if (data != NULL) {
1339                         /* Try to load a file */
1340                         return ucl_include_file (data, len, parser, &params);
1341                 }
1342         }
1343         else {
1344                 if (allow_url && ucl_strnstr (data, "://", len) != NULL) {
1345                         /* Globbing is not used for URL's */
1346                         return ucl_include_url (data, len, parser, &params);
1347                 }
1348
1349                 ip = ucl_object_iterate_new (parser->includepaths);
1350                 while ((param = ucl_object_iterate_safe (ip, true)) != NULL) {
1351                         if (ucl_object_type(param) == UCL_STRING) {
1352                                 snprintf (ipath, sizeof (ipath), "%s/%.*s", ucl_object_tostring(param),
1353                                                 (int)len, data);
1354                                 if ((search = ucl_include_file (ipath, strlen (ipath),
1355                                                 parser, &params))) {
1356                                         if (!params.allow_glob) {
1357                                                 break;
1358                                         }
1359                                 }
1360                         }
1361                 }
1362                 ucl_object_iterate_free (ip);
1363                 if (search == true) {
1364                         return true;
1365                 }
1366                 else {
1367                         ucl_create_err (&parser->err,
1368                                         "cannot find file: %.*s in search path",
1369                                         (int)len, data);
1370                         return false;
1371                 }
1372         }
1373
1374         return false;
1375 }
1376
1377 /**
1378  * Handle include macro
1379  * @param data include data
1380  * @param len length of data
1381  * @param args UCL object representing arguments to the macro
1382  * @param ud user data
1383  * @return
1384  */
1385 bool
1386 ucl_include_handler (const unsigned char *data, size_t len,
1387                 const ucl_object_t *args, void* ud)
1388 {
1389         struct ucl_parser *parser = ud;
1390
1391         return ucl_include_common (data, len, args, parser, false, false);
1392 }
1393
1394 /**
1395  * Handle includes macro
1396  * @param data include data
1397  * @param len length of data
1398  * @param args UCL object representing arguments to the macro
1399  * @param ud user data
1400  * @return
1401  */
1402 bool
1403 ucl_includes_handler (const unsigned char *data, size_t len,
1404                 const ucl_object_t *args, void* ud)
1405 {
1406         struct ucl_parser *parser = ud;
1407
1408         return ucl_include_common (data, len, args, parser, false, true);
1409 }
1410
1411 /**
1412  * Handle tryinclude macro
1413  * @param data include data
1414  * @param len length of data
1415  * @param args UCL object representing arguments to the macro
1416  * @param ud user data
1417  * @return
1418  */
1419 bool
1420 ucl_try_include_handler (const unsigned char *data, size_t len,
1421                 const ucl_object_t *args, void* ud)
1422 {
1423         struct ucl_parser *parser = ud;
1424
1425         return ucl_include_common (data, len, args, parser, true, false);
1426 }
1427
1428 /**
1429  * Handle priority macro
1430  * @param data include data
1431  * @param len length of data
1432  * @param args UCL object representing arguments to the macro
1433  * @param ud user data
1434  * @return
1435  */
1436 bool
1437 ucl_priority_handler (const unsigned char *data, size_t len,
1438                 const ucl_object_t *args, void* ud)
1439 {
1440         struct ucl_parser *parser = ud;
1441         unsigned priority = 255;
1442         const ucl_object_t *param;
1443         bool found = false;
1444         char *value = NULL, *leftover = NULL;
1445         ucl_object_iter_t it = NULL;
1446
1447         if (parser == NULL) {
1448                 return false;
1449         }
1450
1451         /* Process arguments */
1452         if (args != NULL && args->type == UCL_OBJECT) {
1453                 while ((param = ucl_iterate_object (args, &it, true)) != NULL) {
1454                         if (param->type == UCL_INT) {
1455                                 if (strncmp (param->key, "priority", param->keylen) == 0) {
1456                                         priority = ucl_object_toint (param);
1457                                         found = true;
1458                                 }
1459                         }
1460                 }
1461         }
1462
1463         if (len > 0) {
1464                 value = malloc(len + 1);
1465                 ucl_strlcpy(value, (const char *)data, len + 1);
1466                 priority = strtol(value, &leftover, 10);
1467                 if (*leftover != '\0') {
1468                         ucl_create_err (&parser->err, "Invalid priority value in macro: %s",
1469                                 value);
1470                         free(value);
1471                         return false;
1472                 }
1473                 free(value);
1474                 found = true;
1475         }
1476
1477         if (found == true) {
1478                 parser->chunks->priority = priority;
1479                 return true;
1480         }
1481
1482         ucl_create_err (&parser->err, "Unable to parse priority macro");
1483         return false;
1484 }
1485
1486 /**
1487  * Handle load macro
1488  * @param data include data
1489  * @param len length of data
1490  * @param args UCL object representing arguments to the macro
1491  * @param ud user data
1492  * @return
1493  */
1494 bool
1495 ucl_load_handler (const unsigned char *data, size_t len,
1496                 const ucl_object_t *args, void* ud)
1497 {
1498         struct ucl_parser *parser = ud;
1499         const ucl_object_t *param;
1500         ucl_object_t *obj, *old_obj;
1501         ucl_object_iter_t it = NULL;
1502         bool try_load, multiline, test;
1503         const char *target, *prefix;
1504         char *load_file, *tmp;
1505         unsigned char *buf;
1506         size_t buflen;
1507         unsigned priority;
1508         int64_t iv;
1509         ucl_hash_t *container = NULL;
1510         enum ucl_string_flags flags;
1511
1512         /* Default values */
1513         try_load = false;
1514         multiline = false;
1515         test = false;
1516         target = "string";
1517         prefix = NULL;
1518         load_file = NULL;
1519         buf = NULL;
1520         buflen = 0;
1521         priority = 0;
1522         obj = NULL;
1523         old_obj = NULL;
1524         flags = 0;
1525
1526         if (parser == NULL) {
1527                 return false;
1528         }
1529
1530         /* Process arguments */
1531         if (args != NULL && args->type == UCL_OBJECT) {
1532                 while ((param = ucl_iterate_object (args, &it, true)) != NULL) {
1533                         if (param->type == UCL_BOOLEAN) {
1534                                 if (strncmp (param->key, "try", param->keylen) == 0) {
1535                                         try_load = ucl_object_toboolean (param);
1536                                 }
1537                                 else if (strncmp (param->key, "multiline", param->keylen) == 0) {
1538                                         multiline = ucl_object_toboolean (param);
1539                                 }
1540                                 else if (strncmp (param->key, "escape", param->keylen) == 0) {
1541                                         test = ucl_object_toboolean (param);
1542                                         if (test) {
1543                                                 flags |= UCL_STRING_ESCAPE;
1544                                         }
1545                                 }
1546                                 else if (strncmp (param->key, "trim", param->keylen) == 0) {
1547                                         test = ucl_object_toboolean (param);
1548                                         if (test) {
1549                                                 flags |= UCL_STRING_TRIM;
1550                                         }
1551                                 }
1552                         }
1553                         else if (param->type == UCL_STRING) {
1554                                 if (strncmp (param->key, "key", param->keylen) == 0) {
1555                                         prefix = ucl_object_tostring (param);
1556                                 }
1557                                 else if (strncmp (param->key, "target", param->keylen) == 0) {
1558                                         target = ucl_object_tostring (param);
1559                                 }
1560                         }
1561                         else if (param->type == UCL_INT) {
1562                                 if (strncmp (param->key, "priority", param->keylen) == 0) {
1563                                         priority = ucl_object_toint (param);
1564                                 }
1565                         }
1566                 }
1567         }
1568
1569         if (prefix == NULL || strlen(prefix) == 0) {
1570                 ucl_create_err (&parser->err, "No Key specified in load macro");
1571                 return false;
1572         }
1573
1574         if (len > 0) {
1575                 asprintf (&load_file, "%.*s", (int)len, data);
1576                 if (!ucl_fetch_file (load_file, &buf, &buflen, &parser->err, !try_load)) {
1577                         return (try_load || false);
1578                 }
1579
1580                 container = parser->stack->obj->value.ov;
1581                 old_obj = __DECONST (ucl_object_t *, ucl_hash_search (container, prefix, strlen (prefix)));
1582                 if (old_obj != NULL) {
1583                         ucl_create_err (&parser->err, "Key %s already exists", prefix);
1584                         return false;
1585                 }
1586
1587                 if (strcasecmp (target, "string") == 0) {
1588                         obj = ucl_object_fromstring_common (buf, buflen, flags);
1589                         ucl_copy_value_trash (obj);
1590                         if (multiline) {
1591                                 obj->flags |= UCL_OBJECT_MULTILINE;
1592                         }
1593                 }
1594                 else if (strcasecmp (target, "int") == 0) {
1595                         asprintf(&tmp, "%.*s", (int)buflen, buf);
1596                         iv = strtoll(tmp, NULL, 10);
1597                         obj = ucl_object_fromint(iv);
1598                 }
1599
1600                 if (buflen > 0) {
1601                         ucl_munmap (buf, buflen);
1602                 }
1603
1604                 if (obj != NULL) {
1605                         obj->key = prefix;
1606                         obj->keylen = strlen (prefix);
1607                         ucl_copy_key_trash(obj);
1608                         obj->prev = obj;
1609                         obj->next = NULL;
1610                         ucl_object_set_priority (obj, priority);
1611                         container = ucl_hash_insert_object (container, obj,
1612                                         parser->flags & UCL_PARSER_KEY_LOWERCASE);
1613                         parser->stack->obj->value.ov = container;
1614                 }
1615                 return true;
1616         }
1617
1618         ucl_create_err (&parser->err, "Unable to parse load macro");
1619         return false;
1620 }
1621
1622 bool
1623 ucl_inherit_handler (const unsigned char *data, size_t len,
1624                 const ucl_object_t *args, const ucl_object_t *ctx, void* ud)
1625 {
1626         const ucl_object_t *parent, *cur;
1627         ucl_object_t *target, *copy;
1628         ucl_object_iter_t it = NULL;
1629         bool replace = false;
1630         struct ucl_parser *parser = ud;
1631
1632         parent = ucl_object_find_keyl (ctx, data, len);
1633
1634         /* Some sanity checks */
1635         if (parent == NULL || ucl_object_type (parent) != UCL_OBJECT) {
1636                 ucl_create_err (&parser->err, "Unable to find inherited object %*.s",
1637                                 (int)len, data);
1638                 return false;
1639         }
1640
1641         if (parser->stack == NULL || parser->stack->obj == NULL ||
1642                         ucl_object_type (parser->stack->obj) != UCL_OBJECT) {
1643                 ucl_create_err (&parser->err, "Invalid inherit context");
1644                 return false;
1645         }
1646
1647         target = parser->stack->obj;
1648
1649         if (args && (cur = ucl_object_find_key (args, "replace")) != NULL) {
1650                 replace = ucl_object_toboolean (cur);
1651         }
1652
1653         while ((cur = ucl_iterate_object (parent, &it, true))) {
1654                 /* We do not replace existing keys */
1655                 if (!replace && ucl_object_find_keyl (target, cur->key, cur->keylen)) {
1656                         continue;
1657                 }
1658
1659                 copy = ucl_object_copy (cur);
1660
1661                 if (!replace) {
1662                         copy->flags |= UCL_OBJECT_INHERITED;
1663                 }
1664
1665                 ucl_object_insert_key (target, copy, copy->key,
1666                                 copy->keylen, false);
1667         }
1668
1669         return true;
1670 }
1671
1672 bool
1673 ucl_parser_set_filevars (struct ucl_parser *parser, const char *filename, bool need_expand)
1674 {
1675         char realbuf[PATH_MAX], *curdir;
1676
1677         if (filename != NULL) {
1678                 if (need_expand) {
1679                         if (ucl_realpath (filename, realbuf) == NULL) {
1680                                 return false;
1681                         }
1682                 }
1683                 else {
1684                         ucl_strlcpy (realbuf, filename, sizeof (realbuf));
1685                 }
1686
1687                 /* Define variables */
1688                 ucl_parser_register_variable (parser, "FILENAME", realbuf);
1689                 curdir = dirname (realbuf);
1690                 ucl_parser_register_variable (parser, "CURDIR", curdir);
1691         }
1692         else {
1693                 /* Set everything from the current dir */
1694                 curdir = getcwd (realbuf, sizeof (realbuf));
1695                 ucl_parser_register_variable (parser, "FILENAME", "undef");
1696                 ucl_parser_register_variable (parser, "CURDIR", curdir);
1697         }
1698
1699         return true;
1700 }
1701
1702 bool
1703 ucl_parser_add_file_priority (struct ucl_parser *parser, const char *filename,
1704                 unsigned priority)
1705 {
1706         unsigned char *buf;
1707         size_t len;
1708         bool ret;
1709         char realbuf[PATH_MAX];
1710
1711         if (ucl_realpath (filename, realbuf) == NULL) {
1712                 ucl_create_err (&parser->err, "cannot open file %s: %s",
1713                                 filename,
1714                                 strerror (errno));
1715                 return false;
1716         }
1717
1718         if (!ucl_fetch_file (realbuf, &buf, &len, &parser->err, true)) {
1719                 return false;
1720         }
1721
1722         if (parser->cur_file) {
1723                 free (parser->cur_file);
1724         }
1725         parser->cur_file = strdup (realbuf);
1726         ucl_parser_set_filevars (parser, realbuf, false);
1727         ret = ucl_parser_add_chunk_priority (parser, buf, len, priority);
1728
1729         if (len > 0) {
1730                 ucl_munmap (buf, len);
1731         }
1732
1733         return ret;
1734 }
1735
1736 bool
1737 ucl_parser_add_file (struct ucl_parser *parser, const char *filename)
1738 {
1739         if (parser == NULL) {
1740                 return false;
1741         }
1742
1743         return ucl_parser_add_file_priority(parser, filename,
1744                         parser->default_priority);
1745 }
1746
1747 bool
1748 ucl_parser_add_fd_priority (struct ucl_parser *parser, int fd,
1749                 unsigned priority)
1750 {
1751         unsigned char *buf;
1752         size_t len;
1753         bool ret;
1754         struct stat st;
1755
1756         if (fstat (fd, &st) == -1) {
1757                 ucl_create_err (&parser->err, "cannot stat fd %d: %s",
1758                         fd, strerror (errno));
1759                 return false;
1760         }
1761         if ((buf = ucl_mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
1762                 ucl_create_err (&parser->err, "cannot mmap fd %d: %s",
1763                         fd, strerror (errno));
1764                 return false;
1765         }
1766
1767         if (parser->cur_file) {
1768                 free (parser->cur_file);
1769         }
1770         parser->cur_file = NULL;
1771         len = st.st_size;
1772         ret = ucl_parser_add_chunk_priority (parser, buf, len, priority);
1773
1774         if (len > 0) {
1775                 ucl_munmap (buf, len);
1776         }
1777
1778         return ret;
1779 }
1780
1781 bool
1782 ucl_parser_add_fd (struct ucl_parser *parser, int fd)
1783 {
1784         if (parser == NULL) {
1785                 return false;
1786         }
1787
1788         return ucl_parser_add_fd_priority(parser, fd, parser->default_priority);
1789 }
1790
1791 size_t
1792 ucl_strlcpy (char *dst, const char *src, size_t siz)
1793 {
1794         char *d = dst;
1795         const char *s = src;
1796         size_t n = siz;
1797
1798         /* Copy as many bytes as will fit */
1799         if (n != 0) {
1800                 while (--n != 0) {
1801                         if ((*d++ = *s++) == '\0') {
1802                                 break;
1803                         }
1804                 }
1805         }
1806
1807         if (n == 0 && siz != 0) {
1808                 *d = '\0';
1809         }
1810
1811         return (s - src - 1);    /* count does not include NUL */
1812 }
1813
1814 size_t
1815 ucl_strlcpy_unsafe (char *dst, const char *src, size_t siz)
1816 {
1817         memcpy (dst, src, siz - 1);
1818         dst[siz - 1] = '\0';
1819
1820         return siz - 1;
1821 }
1822
1823 size_t
1824 ucl_strlcpy_tolower (char *dst, const char *src, size_t siz)
1825 {
1826         char *d = dst;
1827         const char *s = src;
1828         size_t n = siz;
1829
1830         /* Copy as many bytes as will fit */
1831         if (n != 0) {
1832                 while (--n != 0) {
1833                         if ((*d++ = tolower (*s++)) == '\0') {
1834                                 break;
1835                         }
1836                 }
1837         }
1838
1839         if (n == 0 && siz != 0) {
1840                 *d = '\0';
1841         }
1842
1843         return (s - src);    /* count does not include NUL */
1844 }
1845
1846 /*
1847  * Find the first occurrence of find in s
1848  */
1849 char *
1850 ucl_strnstr (const char *s, const char *find, int len)
1851 {
1852         char c, sc;
1853         int mlen;
1854
1855         if ((c = *find++) != 0) {
1856                 mlen = strlen (find);
1857                 do {
1858                         do {
1859                                 if ((sc = *s++) == 0 || len-- == 0)
1860                                         return (NULL);
1861                         } while (sc != c);
1862                 } while (strncmp (s, find, mlen) != 0);
1863                 s--;
1864         }
1865         return ((char *)s);
1866 }
1867
1868 /*
1869  * Find the first occurrence of find in s, ignore case.
1870  */
1871 char *
1872 ucl_strncasestr (const char *s, const char *find, int len)
1873 {
1874         char c, sc;
1875         int mlen;
1876
1877         if ((c = *find++) != 0) {
1878                 c = tolower (c);
1879                 mlen = strlen (find);
1880                 do {
1881                         do {
1882                                 if ((sc = *s++) == 0 || len-- == 0)
1883                                         return (NULL);
1884                         } while (tolower (sc) != c);
1885                 } while (strncasecmp (s, find, mlen) != 0);
1886                 s--;
1887         }
1888         return ((char *)s);
1889 }
1890
1891 ucl_object_t *
1892 ucl_object_fromstring_common (const char *str, size_t len, enum ucl_string_flags flags)
1893 {
1894         ucl_object_t *obj;
1895         const char *start, *end, *p, *pos;
1896         char *dst, *d;
1897         size_t escaped_len;
1898
1899         if (str == NULL) {
1900                 return NULL;
1901         }
1902
1903         obj = ucl_object_new ();
1904         if (obj) {
1905                 if (len == 0) {
1906                         len = strlen (str);
1907                 }
1908                 if (flags & UCL_STRING_TRIM) {
1909                         /* Skip leading spaces */
1910                         for (start = str; (size_t)(start - str) < len; start ++) {
1911                                 if (!ucl_test_character (*start, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
1912                                         break;
1913                                 }
1914                         }
1915                         /* Skip trailing spaces */
1916                         for (end = str + len - 1; end > start; end --) {
1917                                 if (!ucl_test_character (*end, UCL_CHARACTER_WHITESPACE_UNSAFE)) {
1918                                         break;
1919                                 }
1920                         }
1921                         end ++;
1922                 }
1923                 else {
1924                         start = str;
1925                         end = str + len;
1926                 }
1927
1928                 obj->type = UCL_STRING;
1929                 if (flags & UCL_STRING_ESCAPE) {
1930                         for (p = start, escaped_len = 0; p < end; p ++, escaped_len ++) {
1931                                 if (ucl_test_character (*p, UCL_CHARACTER_JSON_UNSAFE)) {
1932                                         escaped_len ++;
1933                                 }
1934                         }
1935                         dst = malloc (escaped_len + 1);
1936                         if (dst != NULL) {
1937                                 for (p = start, d = dst; p < end; p ++, d ++) {
1938                                         if (ucl_test_character (*p, UCL_CHARACTER_JSON_UNSAFE)) {
1939                                                 switch (*p) {
1940                                                 case '\n':
1941                                                         *d++ = '\\';
1942                                                         *d = 'n';
1943                                                         break;
1944                                                 case '\r':
1945                                                         *d++ = '\\';
1946                                                         *d = 'r';
1947                                                         break;
1948                                                 case '\b':
1949                                                         *d++ = '\\';
1950                                                         *d = 'b';
1951                                                         break;
1952                                                 case '\t':
1953                                                         *d++ = '\\';
1954                                                         *d = 't';
1955                                                         break;
1956                                                 case '\f':
1957                                                         *d++ = '\\';
1958                                                         *d = 'f';
1959                                                         break;
1960                                                 case '\\':
1961                                                         *d++ = '\\';
1962                                                         *d = '\\';
1963                                                         break;
1964                                                 case '"':
1965                                                         *d++ = '\\';
1966                                                         *d = '"';
1967                                                         break;
1968                                                 }
1969                                         }
1970                                         else {
1971                                                 *d = *p;
1972                                         }
1973                                 }
1974                                 *d = '\0';
1975                                 obj->value.sv = dst;
1976                                 obj->trash_stack[UCL_TRASH_VALUE] = dst;
1977                                 obj->len = escaped_len;
1978                         }
1979                 }
1980                 else {
1981                         dst = malloc (end - start + 1);
1982                         if (dst != NULL) {
1983                                 ucl_strlcpy_unsafe (dst, start, end - start + 1);
1984                                 obj->value.sv = dst;
1985                                 obj->trash_stack[UCL_TRASH_VALUE] = dst;
1986                                 obj->len = end - start;
1987                         }
1988                 }
1989                 if ((flags & UCL_STRING_PARSE) && dst != NULL) {
1990                         /* Parse what we have */
1991                         if (flags & UCL_STRING_PARSE_BOOLEAN) {
1992                                 if (!ucl_maybe_parse_boolean (obj, dst, obj->len) && (flags & UCL_STRING_PARSE_NUMBER)) {
1993                                         ucl_maybe_parse_number (obj, dst, dst + obj->len, &pos,
1994                                                         flags & UCL_STRING_PARSE_DOUBLE,
1995                                                         flags & UCL_STRING_PARSE_BYTES,
1996                                                         flags & UCL_STRING_PARSE_TIME);
1997                                 }
1998                         }
1999                         else {
2000                                 ucl_maybe_parse_number (obj, dst, dst + obj->len, &pos,
2001                                                 flags & UCL_STRING_PARSE_DOUBLE,
2002                                                 flags & UCL_STRING_PARSE_BYTES,
2003                                                 flags & UCL_STRING_PARSE_TIME);
2004                         }
2005                 }
2006         }
2007
2008         return obj;
2009 }
2010
2011 static bool
2012 ucl_object_insert_key_common (ucl_object_t *top, ucl_object_t *elt,
2013                 const char *key, size_t keylen, bool copy_key, bool merge, bool replace)
2014 {
2015         ucl_object_t *found, *tmp;
2016         const ucl_object_t *cur;
2017         ucl_object_iter_t it = NULL;
2018         const char *p;
2019         int ret = true;
2020
2021         if (elt == NULL || key == NULL) {
2022                 return false;
2023         }
2024
2025         if (top == NULL) {
2026                 return false;
2027         }
2028
2029         if (top->type != UCL_OBJECT) {
2030                 /* It is possible to convert NULL type to an object */
2031                 if (top->type == UCL_NULL) {
2032                         top->type = UCL_OBJECT;
2033                 }
2034                 else {
2035                         /* Refuse converting of other object types */
2036                         return false;
2037                 }
2038         }
2039
2040         if (top->value.ov == NULL) {
2041                 top->value.ov = ucl_hash_create (false);
2042         }
2043
2044         if (keylen == 0) {
2045                 keylen = strlen (key);
2046         }
2047
2048         for (p = key; p < key + keylen; p ++) {
2049                 if (ucl_test_character (*p, UCL_CHARACTER_UCL_UNSAFE)) {
2050                         elt->flags |= UCL_OBJECT_NEED_KEY_ESCAPE;
2051                         break;
2052                 }
2053         }
2054
2055         /* workaround for some use cases */
2056         if (elt->trash_stack[UCL_TRASH_KEY] != NULL &&
2057                         key != (const char *)elt->trash_stack[UCL_TRASH_KEY]) {
2058                 /* Remove copied key */
2059                 free (elt->trash_stack[UCL_TRASH_KEY]);
2060                 elt->trash_stack[UCL_TRASH_KEY] = NULL;
2061                 elt->flags &= ~UCL_OBJECT_ALLOCATED_KEY;
2062         }
2063
2064         elt->key = key;
2065         elt->keylen = keylen;
2066
2067         if (copy_key) {
2068                 ucl_copy_key_trash (elt);
2069         }
2070
2071         found = __DECONST (ucl_object_t *, ucl_hash_search_obj (top->value.ov, elt));
2072
2073         if (found == NULL) {
2074                 top->value.ov = ucl_hash_insert_object (top->value.ov, elt, false);
2075                 top->len ++;
2076                 if (replace) {
2077                         ret = false;
2078                 }
2079         }
2080         else {
2081                 if (replace) {
2082                         ucl_hash_replace (top->value.ov, found, elt);
2083                         ucl_object_unref (found);
2084                 }
2085                 else if (merge) {
2086                         if (found->type != UCL_OBJECT && elt->type == UCL_OBJECT) {
2087                                 /* Insert old elt to new one */
2088                                 ucl_object_insert_key_common (elt, found, found->key,
2089                                                 found->keylen, copy_key, false, false);
2090                                 ucl_hash_delete (top->value.ov, found);
2091                                 top->value.ov = ucl_hash_insert_object (top->value.ov, elt, false);
2092                         }
2093                         else if (found->type == UCL_OBJECT && elt->type != UCL_OBJECT) {
2094                                 /* Insert new to old */
2095                                 ucl_object_insert_key_common (found, elt, elt->key,
2096                                                 elt->keylen, copy_key, false, false);
2097                         }
2098                         else if (found->type == UCL_OBJECT && elt->type == UCL_OBJECT) {
2099                                 /* Mix two hashes */
2100                                 while ((cur = ucl_iterate_object (elt, &it, true)) != NULL) {
2101                                         tmp = ucl_object_ref (cur);
2102                                         ucl_object_insert_key_common (found, tmp, cur->key,
2103                                                         cur->keylen, copy_key, false, false);
2104                                 }
2105                                 ucl_object_unref (elt);
2106                         }
2107                         else {
2108                                 /* Just make a list of scalars */
2109                                 DL_APPEND (found, elt);
2110                         }
2111                 }
2112                 else {
2113                         DL_APPEND (found, elt);
2114                 }
2115         }
2116
2117         return ret;
2118 }
2119
2120 bool
2121 ucl_object_delete_keyl (ucl_object_t *top, const char *key, size_t keylen)
2122 {
2123         ucl_object_t *found;
2124
2125         if (top == NULL || key == NULL) {
2126                 return false;
2127         }
2128
2129         found = __DECONST (ucl_object_t *, ucl_object_find_keyl (top, key, keylen));
2130
2131         if (found == NULL) {
2132                 return false;
2133         }
2134
2135         ucl_hash_delete (top->value.ov, found);
2136         ucl_object_unref (found);
2137         top->len --;
2138
2139         return true;
2140 }
2141
2142 bool
2143 ucl_object_delete_key (ucl_object_t *top, const char *key)
2144 {
2145         return ucl_object_delete_keyl (top, key, strlen (key));
2146 }
2147
2148 ucl_object_t*
2149 ucl_object_pop_keyl (ucl_object_t *top, const char *key, size_t keylen)
2150 {
2151         const ucl_object_t *found;
2152
2153         if (top == NULL || key == NULL) {
2154                 return false;
2155         }
2156         found = ucl_object_find_keyl (top, key, keylen);
2157
2158         if (found == NULL) {
2159                 return NULL;
2160         }
2161         ucl_hash_delete (top->value.ov, found);
2162         top->len --;
2163
2164         return __DECONST (ucl_object_t *, found);
2165 }
2166
2167 ucl_object_t*
2168 ucl_object_pop_key (ucl_object_t *top, const char *key)
2169 {
2170         return ucl_object_pop_keyl (top, key, strlen (key));
2171 }
2172
2173 bool
2174 ucl_object_insert_key (ucl_object_t *top, ucl_object_t *elt,
2175                 const char *key, size_t keylen, bool copy_key)
2176 {
2177         return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, false, false);
2178 }
2179
2180 bool
2181 ucl_object_insert_key_merged (ucl_object_t *top, ucl_object_t *elt,
2182                 const char *key, size_t keylen, bool copy_key)
2183 {
2184         return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, true, false);
2185 }
2186
2187 bool
2188 ucl_object_replace_key (ucl_object_t *top, ucl_object_t *elt,
2189                 const char *key, size_t keylen, bool copy_key)
2190 {
2191         return ucl_object_insert_key_common (top, elt, key, keylen, copy_key, false, true);
2192 }
2193
2194 bool
2195 ucl_object_merge (ucl_object_t *top, ucl_object_t *elt, bool copy)
2196 {
2197         ucl_object_t *cur = NULL, *cp = NULL, *found = NULL;
2198         ucl_object_iter_t iter = NULL;
2199
2200         if (top == NULL || top->type != UCL_OBJECT || elt == NULL || elt->type != UCL_OBJECT) {
2201                 return false;
2202         }
2203
2204         /* Mix two hashes */
2205         while ((cur = (ucl_object_t*)ucl_hash_iterate (elt->value.ov, &iter))) {
2206                 if (copy) {
2207                         cp = ucl_object_copy (cur);
2208                 }
2209                 else {
2210                         cp = ucl_object_ref (cur);
2211                 }
2212                 found = __DECONST(ucl_object_t *, ucl_hash_search (top->value.ov, cp->key, cp->keylen));
2213                 if (found == NULL) {
2214                         /* The key does not exist */
2215                         top->value.ov = ucl_hash_insert_object (top->value.ov, cp, false);
2216                         top->len ++;
2217                 }
2218                 else {
2219                         /* The key already exists, replace it */
2220                         ucl_hash_replace (top->value.ov, found, cp);
2221                         ucl_object_unref (found);
2222                 }
2223         }
2224
2225         return true;
2226 }
2227
2228 const ucl_object_t *
2229 ucl_object_find_keyl (const ucl_object_t *obj, const char *key, size_t klen)
2230 {
2231         const ucl_object_t *ret;
2232         ucl_object_t srch;
2233
2234         if (obj == NULL || obj->type != UCL_OBJECT || key == NULL) {
2235                 return NULL;
2236         }
2237
2238         srch.key = key;
2239         srch.keylen = klen;
2240         ret = ucl_hash_search_obj (obj->value.ov, &srch);
2241
2242         return ret;
2243 }
2244
2245 const ucl_object_t *
2246 ucl_object_find_key (const ucl_object_t *obj, const char *key)
2247 {
2248         if (key == NULL) {
2249                 return NULL;
2250         }
2251
2252         return ucl_object_find_keyl (obj, key, strlen (key));
2253 }
2254
2255 const ucl_object_t*
2256 ucl_object_find_any_key (const ucl_object_t *obj,
2257                 const char *key, ...)
2258 {
2259         va_list ap;
2260         const ucl_object_t *ret = NULL;
2261         const char *nk = NULL;
2262
2263         if (obj == NULL || key == NULL) {
2264                 return NULL;
2265         }
2266
2267         ret = ucl_object_find_keyl (obj, key, strlen (key));
2268
2269         if (ret == NULL) {
2270                 va_start (ap, key);
2271
2272                 while (ret == NULL) {
2273                         nk = va_arg (ap, const char *);
2274
2275                         if (nk == NULL) {
2276                                 break;
2277                         }
2278                         else {
2279                                 ret = ucl_object_find_keyl (obj, nk, strlen (nk));
2280                         }
2281                 }
2282
2283                 va_end (ap);
2284         }
2285
2286         return ret;
2287 }
2288
2289 const ucl_object_t*
2290 ucl_iterate_object (const ucl_object_t *obj, ucl_object_iter_t *iter, bool expand_values)
2291 {
2292         const ucl_object_t *elt = NULL;
2293
2294         if (obj == NULL || iter == NULL) {
2295                 return NULL;
2296         }
2297
2298         if (expand_values) {
2299                 switch (obj->type) {
2300                 case UCL_OBJECT:
2301                         return (const ucl_object_t*)ucl_hash_iterate (obj->value.ov, iter);
2302                         break;
2303                 case UCL_ARRAY: {
2304                         unsigned int idx;
2305                         UCL_ARRAY_GET (vec, obj);
2306                         idx = (unsigned int)(uintptr_t)(*iter);
2307
2308                         if (vec != NULL) {
2309                                 while (idx < kv_size (*vec)) {
2310                                         if ((elt = kv_A (*vec, idx)) != NULL) {
2311                                                 idx ++;
2312                                                 break;
2313                                         }
2314                                         idx ++;
2315                                 }
2316                                 *iter = (void *)(uintptr_t)idx;
2317                         }
2318
2319                         return elt;
2320                         break;
2321                 }
2322                 default:
2323                         /* Go to linear iteration */
2324                         break;
2325                 }
2326         }
2327         /* Treat everything as a linear list */
2328         elt = *iter;
2329         if (elt == NULL) {
2330                 elt = obj;
2331         }
2332         else if (elt == obj) {
2333                 return NULL;
2334         }
2335         *iter = __DECONST (void *, elt->next ? elt->next : obj);
2336         return elt;
2337
2338         /* Not reached */
2339         return NULL;
2340 }
2341
2342 const char safe_iter_magic[4] = {'u', 'i', 't', 'e'};
2343 struct ucl_object_safe_iter {
2344         char magic[4]; /* safety check */
2345         const ucl_object_t *impl_it; /* implicit object iteration */
2346         ucl_object_iter_t expl_it; /* explicit iteration */
2347 };
2348
2349 #define UCL_SAFE_ITER(ptr) (struct ucl_object_safe_iter *)(ptr)
2350 #define UCL_SAFE_ITER_CHECK(it) do { \
2351         assert (it != NULL); \
2352         assert (memcmp (it->magic, safe_iter_magic, sizeof (it->magic)) == 0); \
2353  } while (0)
2354
2355 ucl_object_iter_t
2356 ucl_object_iterate_new (const ucl_object_t *obj)
2357 {
2358         struct ucl_object_safe_iter *it;
2359
2360         it = UCL_ALLOC (sizeof (*it));
2361         if (it != NULL) {
2362                 memcpy (it->magic, safe_iter_magic, sizeof (it->magic));
2363                 it->expl_it = NULL;
2364                 it->impl_it = obj;
2365         }
2366
2367         return (ucl_object_iter_t)it;
2368 }
2369
2370
2371 ucl_object_iter_t
2372 ucl_object_iterate_reset (ucl_object_iter_t it, const ucl_object_t *obj)
2373 {
2374         struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
2375
2376         UCL_SAFE_ITER_CHECK (rit);
2377
2378         rit->impl_it = obj;
2379         rit->expl_it = NULL;
2380
2381         return it;
2382 }
2383
2384 const ucl_object_t*
2385 ucl_object_iterate_safe (ucl_object_iter_t it, bool expand_values)
2386 {
2387         struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
2388         const ucl_object_t *ret = NULL;
2389
2390         UCL_SAFE_ITER_CHECK (rit);
2391
2392         if (rit->impl_it == NULL) {
2393                 return NULL;
2394         }
2395
2396         if (rit->impl_it->type == UCL_OBJECT || rit->impl_it->type == UCL_ARRAY) {
2397                 ret = ucl_iterate_object (rit->impl_it, &rit->expl_it, true);
2398
2399                 if (ret == NULL) {
2400                         /* Need to switch to another implicit object in chain */
2401                         rit->impl_it = rit->impl_it->next;
2402                         rit->expl_it = NULL;
2403                         return ucl_object_iterate_safe (it, expand_values);
2404                 }
2405         }
2406         else {
2407                 /* Just iterate over the implicit array */
2408                 ret = rit->impl_it;
2409                 rit->impl_it = rit->impl_it->next;
2410                 if (expand_values) {
2411                         /* We flatten objects if need to expand values */
2412                         if (ret->type == UCL_OBJECT || ret->type == UCL_ARRAY) {
2413                                 return ucl_object_iterate_safe (it, expand_values);
2414                         }
2415                 }
2416         }
2417
2418         return ret;
2419 }
2420
2421 void
2422 ucl_object_iterate_free (ucl_object_iter_t it)
2423 {
2424         struct ucl_object_safe_iter *rit = UCL_SAFE_ITER (it);
2425
2426         UCL_SAFE_ITER_CHECK (rit);
2427
2428         UCL_FREE (sizeof (*rit), it);
2429 }
2430
2431 const ucl_object_t *
2432 ucl_lookup_path (const ucl_object_t *top, const char *path_in) {
2433         return ucl_lookup_path_char (top, path_in, '.');
2434 }
2435
2436
2437 const ucl_object_t *
2438 ucl_lookup_path_char (const ucl_object_t *top, const char *path_in, const char sep) {
2439         const ucl_object_t *o = NULL, *found;
2440         const char *p, *c;
2441         char *err_str;
2442         unsigned index;
2443
2444         if (path_in == NULL || top == NULL) {
2445                 return NULL;
2446         }
2447
2448         found = NULL;
2449         p = path_in;
2450
2451         /* Skip leading dots */
2452         while (*p == sep) {
2453                 p ++;
2454         }
2455
2456         c = p;
2457         while (*p != '\0') {
2458                 p ++;
2459                 if (*p == sep || *p == '\0') {
2460                         if (p > c) {
2461                                 switch (top->type) {
2462                                 case UCL_ARRAY:
2463                                         /* Key should be an int */
2464                                         index = strtoul (c, &err_str, 10);
2465                                         if (err_str != NULL && (*err_str != sep && *err_str != '\0')) {
2466                                                 return NULL;
2467                                         }
2468                                         o = ucl_array_find_index (top, index);
2469                                         break;
2470                                 default:
2471                                         o = ucl_object_find_keyl (top, c, p - c);
2472                                         break;
2473                                 }
2474                                 if (o == NULL) {
2475                                         return NULL;
2476                                 }
2477                                 top = o;
2478                         }
2479                         if (*p != '\0') {
2480                                 c = p + 1;
2481                         }
2482                 }
2483         }
2484         found = o;
2485
2486         return found;
2487 }
2488
2489
2490 ucl_object_t *
2491 ucl_object_new (void)
2492 {
2493         return ucl_object_typed_new (UCL_NULL);
2494 }
2495
2496 ucl_object_t *
2497 ucl_object_typed_new (ucl_type_t type)
2498 {
2499         return ucl_object_new_full (type, 0);
2500 }
2501
2502 ucl_object_t *
2503 ucl_object_new_full (ucl_type_t type, unsigned priority)
2504 {
2505         ucl_object_t *new;
2506
2507         if (type != UCL_USERDATA) {
2508                 new = UCL_ALLOC (sizeof (ucl_object_t));
2509                 if (new != NULL) {
2510                         memset (new, 0, sizeof (ucl_object_t));
2511                         new->ref = 1;
2512                         new->type = (type <= UCL_NULL ? type : UCL_NULL);
2513                         new->next = NULL;
2514                         new->prev = new;
2515                         ucl_object_set_priority (new, priority);
2516
2517                         if (type == UCL_ARRAY) {
2518                                 new->value.av = UCL_ALLOC (sizeof (ucl_array_t));
2519                                 if (new->value.av) {
2520                                         memset (new->value.av, 0, sizeof (ucl_array_t));
2521                                         UCL_ARRAY_GET (vec, new);
2522
2523                                         /* Preallocate some space for arrays */
2524                                         kv_resize (ucl_object_t *, *vec, 8);
2525                                 }
2526                         }
2527                 }
2528         }
2529         else {
2530                 new = ucl_object_new_userdata (NULL, NULL);
2531                 ucl_object_set_priority (new, priority);
2532         }
2533
2534         return new;
2535 }
2536
2537 ucl_object_t*
2538 ucl_object_new_userdata (ucl_userdata_dtor dtor, ucl_userdata_emitter emitter)
2539 {
2540         struct ucl_object_userdata *new;
2541         size_t nsize = sizeof (*new);
2542
2543         new = UCL_ALLOC (nsize);
2544         if (new != NULL) {
2545                 memset (new, 0, nsize);
2546                 new->obj.ref = 1;
2547                 new->obj.type = UCL_USERDATA;
2548                 new->obj.next = NULL;
2549                 new->obj.prev = (ucl_object_t *)new;
2550                 new->dtor = dtor;
2551                 new->emitter = emitter;
2552         }
2553
2554         return (ucl_object_t *)new;
2555 }
2556
2557 ucl_type_t
2558 ucl_object_type (const ucl_object_t *obj)
2559 {
2560         if (obj == NULL) {
2561                 return UCL_NULL;
2562         }
2563
2564         return obj->type;
2565 }
2566
2567 ucl_object_t*
2568 ucl_object_fromstring (const char *str)
2569 {
2570         return ucl_object_fromstring_common (str, 0, UCL_STRING_ESCAPE);
2571 }
2572
2573 ucl_object_t *
2574 ucl_object_fromlstring (const char *str, size_t len)
2575 {
2576         return ucl_object_fromstring_common (str, len, UCL_STRING_ESCAPE);
2577 }
2578
2579 ucl_object_t *
2580 ucl_object_fromint (int64_t iv)
2581 {
2582         ucl_object_t *obj;
2583
2584         obj = ucl_object_new ();
2585         if (obj != NULL) {
2586                 obj->type = UCL_INT;
2587                 obj->value.iv = iv;
2588         }
2589
2590         return obj;
2591 }
2592
2593 ucl_object_t *
2594 ucl_object_fromdouble (double dv)
2595 {
2596         ucl_object_t *obj;
2597
2598         obj = ucl_object_new ();
2599         if (obj != NULL) {
2600                 obj->type = UCL_FLOAT;
2601                 obj->value.dv = dv;
2602         }
2603
2604         return obj;
2605 }
2606
2607 ucl_object_t*
2608 ucl_object_frombool (bool bv)
2609 {
2610         ucl_object_t *obj;
2611
2612         obj = ucl_object_new ();
2613         if (obj != NULL) {
2614                 obj->type = UCL_BOOLEAN;
2615                 obj->value.iv = bv;
2616         }
2617
2618         return obj;
2619 }
2620
2621 bool
2622 ucl_array_append (ucl_object_t *top, ucl_object_t *elt)
2623 {
2624         UCL_ARRAY_GET (vec, top);
2625
2626         if (elt == NULL || top == NULL) {
2627                 return false;
2628         }
2629
2630         if (vec == NULL) {
2631                 vec = UCL_ALLOC (sizeof (*vec));
2632
2633                 if (vec == NULL) {
2634                         return false;
2635                 }
2636
2637                 kv_init (*vec);
2638                 top->value.av = (void *)vec;
2639         }
2640
2641         kv_push (ucl_object_t *, *vec, elt);
2642
2643         top->len ++;
2644
2645         return true;
2646 }
2647
2648 bool
2649 ucl_array_prepend (ucl_object_t *top, ucl_object_t *elt)
2650 {
2651         UCL_ARRAY_GET (vec, top);
2652
2653         if (elt == NULL || top == NULL) {
2654                 return false;
2655         }
2656
2657         if (vec == NULL) {
2658                 vec = UCL_ALLOC (sizeof (*vec));
2659                 kv_init (*vec);
2660                 top->value.av = (void *)vec;
2661                 kv_push (ucl_object_t *, *vec, elt);
2662         }
2663         else {
2664                 /* Slow O(n) algorithm */
2665                 kv_prepend (ucl_object_t *, *vec, elt);
2666         }
2667
2668         top->len ++;
2669
2670         return true;
2671 }
2672
2673 bool
2674 ucl_array_merge (ucl_object_t *top, ucl_object_t *elt, bool copy)
2675 {
2676         unsigned i;
2677         ucl_object_t *cp = NULL;
2678         ucl_object_t **obj;
2679
2680         if (elt == NULL || top == NULL || top->type != UCL_ARRAY || elt->type != UCL_ARRAY) {
2681                 return false;
2682         }
2683
2684         if (copy) {
2685                 cp = ucl_object_copy (elt);
2686         }
2687         else {
2688                 cp = ucl_object_ref (elt);
2689         }
2690
2691         UCL_ARRAY_GET (v1, top);
2692         UCL_ARRAY_GET (v2, cp);
2693
2694         kv_concat (ucl_object_t *, *v1, *v2);
2695
2696         for (i = v2->n; i < v1->n; i ++) {
2697                 obj = &kv_A (*v1, i);
2698                 if (*obj == NULL) {
2699                         continue;
2700                 }
2701                 top->len ++;
2702         }
2703
2704         return true;
2705 }
2706
2707 ucl_object_t *
2708 ucl_array_delete (ucl_object_t *top, ucl_object_t *elt)
2709 {
2710         UCL_ARRAY_GET (vec, top);
2711         ucl_object_t *ret = NULL;
2712         unsigned i;
2713
2714         if (vec == NULL) {
2715                 return NULL;
2716         }
2717
2718         for (i = 0; i < vec->n; i ++) {
2719                 if (kv_A (*vec, i) == elt) {
2720                         kv_del (ucl_object_t *, *vec, i);
2721                         ret = elt;
2722                         top->len --;
2723                         break;
2724                 }
2725         }
2726
2727         return ret;
2728 }
2729
2730 const ucl_object_t *
2731 ucl_array_head (const ucl_object_t *top)
2732 {
2733         UCL_ARRAY_GET (vec, top);
2734
2735         if (vec == NULL || top == NULL || top->type != UCL_ARRAY ||
2736                         top->value.av == NULL) {
2737                 return NULL;
2738         }
2739
2740         return (vec->n > 0 ? vec->a[0] : NULL);
2741 }
2742
2743 const ucl_object_t *
2744 ucl_array_tail (const ucl_object_t *top)
2745 {
2746         UCL_ARRAY_GET (vec, top);
2747
2748         if (top == NULL || top->type != UCL_ARRAY || top->value.av == NULL) {
2749                 return NULL;
2750         }
2751
2752         return (vec->n > 0 ? vec->a[vec->n - 1] : NULL);
2753 }
2754
2755 ucl_object_t *
2756 ucl_array_pop_last (ucl_object_t *top)
2757 {
2758         UCL_ARRAY_GET (vec, top);
2759         ucl_object_t **obj, *ret = NULL;
2760
2761         if (vec != NULL && vec->n > 0) {
2762                 obj = &kv_A (*vec, vec->n - 1);
2763                 ret = *obj;
2764                 kv_del (ucl_object_t *, *vec, vec->n - 1);
2765                 top->len --;
2766         }
2767
2768         return ret;
2769 }
2770
2771 ucl_object_t *
2772 ucl_array_pop_first (ucl_object_t *top)
2773 {
2774         UCL_ARRAY_GET (vec, top);
2775         ucl_object_t **obj, *ret = NULL;
2776
2777         if (vec != NULL && vec->n > 0) {
2778                 obj = &kv_A (*vec, 0);
2779                 ret = *obj;
2780                 kv_del (ucl_object_t *, *vec, 0);
2781                 top->len --;
2782         }
2783
2784         return ret;
2785 }
2786
2787 const ucl_object_t *
2788 ucl_array_find_index (const ucl_object_t *top, unsigned int index)
2789 {
2790         UCL_ARRAY_GET (vec, top);
2791
2792         if (vec != NULL && vec->n > 0 && index < vec->n) {
2793                 return kv_A (*vec, index);
2794         }
2795
2796         return NULL;
2797 }
2798
2799 unsigned int
2800 ucl_array_index_of (ucl_object_t *top, ucl_object_t *elt)
2801 {
2802         UCL_ARRAY_GET (vec, top);
2803         unsigned i;
2804
2805         if (vec == NULL) {
2806                 return (unsigned int)(-1);
2807         }
2808
2809         for (i = 0; i < vec->n; i ++) {
2810                 if (kv_A (*vec, i) == elt) {
2811                         return i;
2812                 }
2813         }
2814
2815         return (unsigned int)(-1);
2816 }
2817
2818 ucl_object_t *
2819 ucl_array_replace_index (ucl_object_t *top, ucl_object_t *elt,
2820         unsigned int index)
2821 {
2822         UCL_ARRAY_GET (vec, top);
2823         ucl_object_t *ret = NULL;
2824
2825         if (vec != NULL && vec->n > 0 && index < vec->n) {
2826                 ret = kv_A (*vec, index);
2827                 kv_A (*vec, index) = elt;
2828         }
2829
2830         return ret;
2831 }
2832
2833 ucl_object_t *
2834 ucl_elt_append (ucl_object_t *head, ucl_object_t *elt)
2835 {
2836
2837         if (head == NULL) {
2838                 elt->next = NULL;
2839                 elt->prev = elt;
2840                 head = elt;
2841         }
2842         else {
2843                 elt->prev = head->prev;
2844                 head->prev->next = elt;
2845                 head->prev = elt;
2846                 elt->next = NULL;
2847         }
2848
2849         return head;
2850 }
2851
2852 bool
2853 ucl_object_todouble_safe (const ucl_object_t *obj, double *target)
2854 {
2855         if (obj == NULL || target == NULL) {
2856                 return false;
2857         }
2858         switch (obj->type) {
2859         case UCL_INT:
2860                 *target = obj->value.iv; /* Probaly could cause overflow */
2861                 break;
2862         case UCL_FLOAT:
2863         case UCL_TIME:
2864                 *target = obj->value.dv;
2865                 break;
2866         default:
2867                 return false;
2868         }
2869
2870         return true;
2871 }
2872
2873 double
2874 ucl_object_todouble (const ucl_object_t *obj)
2875 {
2876         double result = 0.;
2877
2878         ucl_object_todouble_safe (obj, &result);
2879         return result;
2880 }
2881
2882 bool
2883 ucl_object_toint_safe (const ucl_object_t *obj, int64_t *target)
2884 {
2885         if (obj == NULL || target == NULL) {
2886                 return false;
2887         }
2888         switch (obj->type) {
2889         case UCL_INT:
2890                 *target = obj->value.iv;
2891                 break;
2892         case UCL_FLOAT:
2893         case UCL_TIME:
2894                 *target = obj->value.dv; /* Loosing of decimal points */
2895                 break;
2896         default:
2897                 return false;
2898         }
2899
2900         return true;
2901 }
2902
2903 int64_t
2904 ucl_object_toint (const ucl_object_t *obj)
2905 {
2906         int64_t result = 0;
2907
2908         ucl_object_toint_safe (obj, &result);
2909         return result;
2910 }
2911
2912 bool
2913 ucl_object_toboolean_safe (const ucl_object_t *obj, bool *target)
2914 {
2915         if (obj == NULL || target == NULL) {
2916                 return false;
2917         }
2918         switch (obj->type) {
2919         case UCL_BOOLEAN:
2920                 *target = (obj->value.iv == true);
2921                 break;
2922         default:
2923                 return false;
2924         }
2925
2926         return true;
2927 }
2928
2929 bool
2930 ucl_object_toboolean (const ucl_object_t *obj)
2931 {
2932         bool result = false;
2933
2934         ucl_object_toboolean_safe (obj, &result);
2935         return result;
2936 }
2937
2938 bool
2939 ucl_object_tostring_safe (const ucl_object_t *obj, const char **target)
2940 {
2941         if (obj == NULL || target == NULL) {
2942                 return false;
2943         }
2944
2945         switch (obj->type) {
2946         case UCL_STRING:
2947                 if (!(obj->flags & UCL_OBJECT_BINARY)) {
2948                         *target = ucl_copy_value_trash (obj);
2949                 }
2950                 break;
2951         default:
2952                 return false;
2953         }
2954
2955         return true;
2956 }
2957
2958 const char *
2959 ucl_object_tostring (const ucl_object_t *obj)
2960 {
2961         const char *result = NULL;
2962
2963         ucl_object_tostring_safe (obj, &result);
2964         return result;
2965 }
2966
2967 const char *
2968 ucl_object_tostring_forced (const ucl_object_t *obj)
2969 {
2970         /* TODO: For binary strings we might encode string here */
2971         if (!(obj->flags & UCL_OBJECT_BINARY)) {
2972                 return ucl_copy_value_trash (obj);
2973         }
2974
2975         return NULL;
2976 }
2977
2978 bool
2979 ucl_object_tolstring_safe (const ucl_object_t *obj, const char **target, size_t *tlen)
2980 {
2981         if (obj == NULL || target == NULL) {
2982                 return false;
2983         }
2984         switch (obj->type) {
2985         case UCL_STRING:
2986                 *target = obj->value.sv;
2987                 if (tlen != NULL) {
2988                         *tlen = obj->len;
2989                 }
2990                 break;
2991         default:
2992                 return false;
2993         }
2994
2995         return true;
2996 }
2997
2998 const char *
2999 ucl_object_tolstring (const ucl_object_t *obj, size_t *tlen)
3000 {
3001         const char *result = NULL;
3002
3003         ucl_object_tolstring_safe (obj, &result, tlen);
3004         return result;
3005 }
3006
3007 const char *
3008 ucl_object_key (const ucl_object_t *obj)
3009 {
3010         return ucl_copy_key_trash (obj);
3011 }
3012
3013 const char *
3014 ucl_object_keyl (const ucl_object_t *obj, size_t *len)
3015 {
3016         if (len == NULL || obj == NULL) {
3017                 return NULL;
3018         }
3019         *len = obj->keylen;
3020         return obj->key;
3021 }
3022
3023 ucl_object_t *
3024 ucl_object_ref (const ucl_object_t *obj)
3025 {
3026         ucl_object_t *res = NULL;
3027
3028         if (obj != NULL) {
3029                 if (obj->flags & UCL_OBJECT_EPHEMERAL) {
3030                         /*
3031                          * Use deep copy for ephemeral objects, note that its refcount
3032                          * is NOT increased, since ephemeral objects does not need refcount
3033                          * at all
3034                          */
3035                         res = ucl_object_copy (obj);
3036                 }
3037                 else {
3038                         res = __DECONST (ucl_object_t *, obj);
3039 #ifdef HAVE_ATOMIC_BUILTINS
3040                         (void)__sync_add_and_fetch (&res->ref, 1);
3041 #else
3042                         res->ref ++;
3043 #endif
3044                 }
3045         }
3046         return res;
3047 }
3048
3049 static ucl_object_t *
3050 ucl_object_copy_internal (const ucl_object_t *other, bool allow_array)
3051 {
3052
3053         ucl_object_t *new;
3054         ucl_object_iter_t it = NULL;
3055         const ucl_object_t *cur;
3056
3057         new = malloc (sizeof (*new));
3058
3059         if (new != NULL) {
3060                 memcpy (new, other, sizeof (*new));
3061                 if (other->flags & UCL_OBJECT_EPHEMERAL) {
3062                         /* Copied object is always non ephemeral */
3063                         new->flags &= ~UCL_OBJECT_EPHEMERAL;
3064                 }
3065                 new->ref = 1;
3066                 /* Unlink from others */
3067                 new->next = NULL;
3068                 new->prev = new;
3069
3070                 /* deep copy of values stored */
3071                 if (other->trash_stack[UCL_TRASH_KEY] != NULL) {
3072                         new->trash_stack[UCL_TRASH_KEY] =
3073                                         strdup (other->trash_stack[UCL_TRASH_KEY]);
3074                         if (other->key == (const char *)other->trash_stack[UCL_TRASH_KEY]) {
3075                                 new->key = new->trash_stack[UCL_TRASH_KEY];
3076                         }
3077                 }
3078                 if (other->trash_stack[UCL_TRASH_VALUE] != NULL) {
3079                         new->trash_stack[UCL_TRASH_VALUE] =
3080                                         strdup (other->trash_stack[UCL_TRASH_VALUE]);
3081                         if (new->type == UCL_STRING) {
3082                                 new->value.sv = new->trash_stack[UCL_TRASH_VALUE];
3083                         }
3084                 }
3085
3086                 if (other->type == UCL_ARRAY || other->type == UCL_OBJECT) {
3087                         /* reset old value */
3088                         memset (&new->value, 0, sizeof (new->value));
3089
3090                         while ((cur = ucl_iterate_object (other, &it, true)) != NULL) {
3091                                 if (other->type == UCL_ARRAY) {
3092                                         ucl_array_append (new, ucl_object_copy_internal (cur, false));
3093                                 }
3094                                 else {
3095                                         ucl_object_t *cp = ucl_object_copy_internal (cur, true);
3096                                         if (cp != NULL) {
3097                                                 ucl_object_insert_key (new, cp, cp->key, cp->keylen,
3098                                                                 false);
3099                                         }
3100                                 }
3101                         }
3102                 }
3103                 else if (allow_array && other->next != NULL) {
3104                         LL_FOREACH (other->next, cur) {
3105                                 ucl_object_t *cp = ucl_object_copy_internal (cur, false);
3106                                 if (cp != NULL) {
3107                                         DL_APPEND (new, cp);
3108                                 }
3109                         }
3110                 }
3111         }
3112
3113         return new;
3114 }
3115
3116 ucl_object_t *
3117 ucl_object_copy (const ucl_object_t *other)
3118 {
3119         return ucl_object_copy_internal (other, true);
3120 }
3121
3122 void
3123 ucl_object_unref (ucl_object_t *obj)
3124 {
3125         if (obj != NULL) {
3126 #ifdef HAVE_ATOMIC_BUILTINS
3127                 unsigned int rc = __sync_sub_and_fetch (&obj->ref, 1);
3128                 if (rc == 0) {
3129 #else
3130                 if (--obj->ref == 0) {
3131 #endif
3132                         ucl_object_free_internal (obj, true, ucl_object_dtor_unref);
3133                 }
3134         }
3135 }
3136
3137 int
3138 ucl_object_compare (const ucl_object_t *o1, const ucl_object_t *o2)
3139 {
3140         const ucl_object_t *it1, *it2;
3141         ucl_object_iter_t iter = NULL;
3142         int ret = 0;
3143
3144         if (o1->type != o2->type) {
3145                 return (o1->type) - (o2->type);
3146         }
3147
3148         switch (o1->type) {
3149         case UCL_STRING:
3150                 if (o1->len == o2->len && o1->len > 0) {
3151                         ret = strcmp (ucl_object_tostring(o1), ucl_object_tostring(o2));
3152                 }
3153                 else {
3154                         ret = o1->len - o2->len;
3155                 }
3156                 break;
3157         case UCL_FLOAT:
3158         case UCL_INT:
3159         case UCL_TIME:
3160                 ret = ucl_object_todouble (o1) - ucl_object_todouble (o2);
3161                 break;
3162         case UCL_BOOLEAN:
3163                 ret = ucl_object_toboolean (o1) - ucl_object_toboolean (o2);
3164                 break;
3165         case UCL_ARRAY:
3166                 if (o1->len == o2->len && o1->len > 0) {
3167                         UCL_ARRAY_GET (vec1, o1);
3168                         UCL_ARRAY_GET (vec2, o2);
3169                         unsigned i;
3170
3171                         /* Compare all elements in both arrays */
3172                         for (i = 0; i < vec1->n; i ++) {
3173                                 it1 = kv_A (*vec1, i);
3174                                 it2 = kv_A (*vec2, i);
3175
3176                                 if (it1 == NULL && it2 != NULL) {
3177                                         return -1;
3178                                 }
3179                                 else if (it2 == NULL && it1 != NULL) {
3180                                         return 1;
3181                                 }
3182                                 else if (it1 != NULL && it2 != NULL) {
3183                                         ret = ucl_object_compare (it1, it2);
3184                                         if (ret != 0) {
3185                                                 break;
3186                                         }
3187                                 }
3188                         }
3189                 }
3190                 else {
3191                         ret = o1->len - o2->len;
3192                 }
3193                 break;
3194         case UCL_OBJECT:
3195                 if (o1->len == o2->len && o1->len > 0) {
3196                         while ((it1 = ucl_iterate_object (o1, &iter, true)) != NULL) {
3197                                 it2 = ucl_object_find_key (o2, ucl_object_key (it1));
3198                                 if (it2 == NULL) {
3199                                         ret = 1;
3200                                         break;
3201                                 }
3202                                 ret = ucl_object_compare (it1, it2);
3203                                 if (ret != 0) {
3204                                         break;
3205                                 }
3206                         }
3207                 }
3208                 else {
3209                         ret = o1->len - o2->len;
3210                 }
3211                 break;
3212         default:
3213                 ret = 0;
3214                 break;
3215         }
3216
3217         return ret;
3218 }
3219
3220 void
3221 ucl_object_array_sort (ucl_object_t *ar,
3222                 int (*cmp)(const ucl_object_t **o1, const ucl_object_t **o2))
3223 {
3224         UCL_ARRAY_GET (vec, ar);
3225
3226         if (cmp == NULL || ar == NULL || ar->type != UCL_ARRAY) {
3227                 return;
3228         }
3229
3230         qsort (vec->a, vec->n, sizeof (ucl_object_t *),
3231                         (int (*)(const void *, const void *))cmp);
3232 }
3233
3234 #define PRIOBITS 4
3235
3236 unsigned int
3237 ucl_object_get_priority (const ucl_object_t *obj)
3238 {
3239         if (obj == NULL) {
3240                 return 0;
3241         }
3242
3243         return (obj->flags >> ((sizeof (obj->flags) * NBBY) - PRIOBITS));
3244 }
3245
3246 void
3247 ucl_object_set_priority (ucl_object_t *obj,
3248                 unsigned int priority)
3249 {
3250         if (obj != NULL) {
3251                 priority &= (0x1 << PRIOBITS) - 1;
3252                 priority <<= ((sizeof (obj->flags) * NBBY) - PRIOBITS);
3253                 priority |= obj->flags & ((1 << ((sizeof (obj->flags) * NBBY) -
3254                                 PRIOBITS)) - 1);
3255                 obj->flags = priority;
3256         }
3257 }