lib/libzfs/libzfs_sendrecv.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
  25  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  26  * Copyright (c) 2012 Pawel Jakub Dawidek <pawel@dawidek.net>.
  27  * All rights reserved
  28  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  29  * Copyright 2015, OmniTI Computer Consulting, Inc. All rights reserved.
  30  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>
  31  * Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
  32  */
  33
  34 #include <assert.h>
  35 #include <ctype.h>
  36 #include <errno.h>
  37 #include <libintl.h>
  38 #include <stdio.h>
  39 #include <stdlib.h>
  40 #include <strings.h>
  41 #include <unistd.h>
  42 #include <stddef.h>
  43 #include <fcntl.h>
  44 #include <sys/mount.h>
  45 #include <sys/mntent.h>
  46 #include <sys/mnttab.h>
  47 #include <sys/avl.h>
  48 #include <sys/debug.h>
  49 #include <sys/stat.h>
  50 #include <stddef.h>
  51 #include <pthread.h>
  52 #include <umem.h>
  53 #include <time.h>
  54
  55 #include <libzfs.h>
  56 #include <libzfs_core.h>
  57 #include <libzutil.h>
  58
  59 #include "zfs_namecheck.h"
  60 #include "zfs_prop.h"
  61 #include "zfs_fletcher.h"
  62 #include "libzfs_impl.h"
  63 #include <zlib.h>
  64 #include <sys/zio_checksum.h>
  65 #include <sys/dsl_crypt.h>
  66 #include <sys/ddt.h>
  67 #include <sys/socket.h>
  68 #include <sys/sha2.h>
  69
  70 /* in libzfs_dataset.c */
  71 extern void zfs_setprop_error(libzfs_handle_t *, zfs_prop_t, int, char *);
  72
  73 static int zfs_receive_impl(libzfs_handle_t *, const char *, const char *,
  74     recvflags_t *, int, const char *, nvlist_t *, avl_tree_t *, char **, int,
  75     uint64_t *, const char *, nvlist_t *);
  76 static int guid_to_name(libzfs_handle_t *, const char *,
  77     uint64_t, boolean_t, char *);
  78
  79 static const zio_cksum_t zero_cksum = { { 0 } };
  80
  81 typedef struct dedup_arg {
  82         int     inputfd;
  83         int     outputfd;
  84         libzfs_handle_t  *dedup_hdl;
  85 } dedup_arg_t;
  86
  87 typedef struct progress_arg {
  88         zfs_handle_t *pa_zhp;
  89         int pa_fd;
  90         boolean_t pa_parsable;
  91 } progress_arg_t;
  92
  93 typedef struct dataref {
  94         uint64_t ref_guid;
  95         uint64_t ref_object;
  96         uint64_t ref_offset;
  97 } dataref_t;
  98
  99 typedef struct dedup_entry {
 100         struct dedup_entry      *dde_next;
 101         zio_cksum_t dde_chksum;
 102         uint64_t dde_prop;
 103         dataref_t dde_ref;
 104 } dedup_entry_t;
 105
 106 #define MAX_DDT_PHYSMEM_PERCENT         20
 107 #define SMALLEST_POSSIBLE_MAX_DDT_MB            128
 108
 109 typedef struct dedup_table {
 110         dedup_entry_t   **dedup_hash_array;
 111         umem_cache_t    *ddecache;
 112         uint64_t        max_ddt_size;  /* max dedup table size in bytes */
 113         uint64_t        cur_ddt_size;  /* current dedup table size in bytes */
 114         uint64_t        ddt_count;
 115         int             numhashbits;
 116         boolean_t       ddt_full;
 117 } dedup_table_t;
 118
 119 static int
 120 high_order_bit(uint64_t n)
 121 {
 122         int count;
 123
 124         for (count = 0; n != 0; count++)
 125                 n >>= 1;
 126         return (count);
 127 }
 128
 129 static size_t
 130 ssread(void *buf, size_t len, FILE *stream)
 131 {
 132         size_t outlen;
 133
 134         if ((outlen = fread(buf, len, 1, stream)) == 0)
 135                 return (0);
 136
 137         return (outlen);
 138 }
 139
 140 static void
 141 ddt_hash_append(libzfs_handle_t *hdl, dedup_table_t *ddt, dedup_entry_t **ddepp,
 142     zio_cksum_t *cs, uint64_t prop, dataref_t *dr)
 143 {
 144         dedup_entry_t   *dde;
 145
 146         if (ddt->cur_ddt_size >= ddt->max_ddt_size) {
 147                 if (ddt->ddt_full == B_FALSE) {
 148                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
 149                             "Dedup table full.  Deduplication will continue "
 150                             "with existing table entries"));
 151                         ddt->ddt_full = B_TRUE;
 152                 }
 153                 return;
 154         }
 155
 156         if ((dde = umem_cache_alloc(ddt->ddecache, UMEM_DEFAULT))
 157             != NULL) {
 158                 assert(*ddepp == NULL);
 159                 dde->dde_next = NULL;
 160                 dde->dde_chksum = *cs;
 161                 dde->dde_prop = prop;
 162                 dde->dde_ref = *dr;
 163                 *ddepp = dde;
 164                 ddt->cur_ddt_size += sizeof (dedup_entry_t);
 165                 ddt->ddt_count++;
 166         }
 167 }
 168
 169 /*
 170  * Using the specified dedup table, do a lookup for an entry with
 171  * the checksum cs.  If found, return the block's reference info
 172  * in *dr. Otherwise, insert a new entry in the dedup table, using
 173  * the reference information specified by *dr.
 174  *
 175  * return value:  true - entry was found
 176  *                false - entry was not found
 177  */
 178 static boolean_t
 179 ddt_update(libzfs_handle_t *hdl, dedup_table_t *ddt, zio_cksum_t *cs,
 180     uint64_t prop, dataref_t *dr)
 181 {
 182         uint32_t hashcode;
 183         dedup_entry_t **ddepp;
 184
 185         hashcode = BF64_GET(cs->zc_word[0], 0, ddt->numhashbits);
 186
 187         for (ddepp = &(ddt->dedup_hash_array[hashcode]); *ddepp != NULL;
 188             ddepp = &((*ddepp)->dde_next)) {
 189                 if (ZIO_CHECKSUM_EQUAL(((*ddepp)->dde_chksum), *cs) &&
 190                     (*ddepp)->dde_prop == prop) {
 191                         *dr = (*ddepp)->dde_ref;
 192                         return (B_TRUE);
 193                 }
 194         }
 195         ddt_hash_append(hdl, ddt, ddepp, cs, prop, dr);
 196         return (B_FALSE);
 197 }
 198
 199 static int
 200 dump_record(dmu_replay_record_t *drr, void *payload, int payload_len,
 201     zio_cksum_t *zc, int outfd)
 202 {
 203         ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
 204             ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t));
 205         fletcher_4_incremental_native(drr,
 206             offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum), zc);
 207         if (drr->drr_type != DRR_BEGIN) {
 208                 ASSERT(ZIO_CHECKSUM_IS_ZERO(&drr->drr_u.
 209                     drr_checksum.drr_checksum));
 210                 drr->drr_u.drr_checksum.drr_checksum = *zc;
 211         }
 212         fletcher_4_incremental_native(&drr->drr_u.drr_checksum.drr_checksum,
 213             sizeof (zio_cksum_t), zc);
 214         if (write(outfd, drr, sizeof (*drr)) == -1)
 215                 return (errno);
 216         if (payload_len != 0) {
 217                 fletcher_4_incremental_native(payload, payload_len, zc);
 218                 if (write(outfd, payload, payload_len) == -1)
 219                         return (errno);
 220         }
 221         return (0);
 222 }
 223
 224 /*
 225  * This function is started in a separate thread when the dedup option
 226  * has been requested.  The main send thread determines the list of
 227  * snapshots to be included in the send stream and makes the ioctl calls
 228  * for each one.  But instead of having the ioctl send the output to the
 229  * the output fd specified by the caller of zfs_send()), the
 230  * ioctl is told to direct the output to a pipe, which is read by the
 231  * alternate thread running THIS function.  This function does the
 232  * dedup'ing by:
 233  *  1. building a dedup table (the DDT)
 234  *  2. doing checksums on each data block and inserting a record in the DDT
 235  *  3. looking for matching checksums, and
 236  *  4.  sending a DRR_WRITE_BYREF record instead of a write record whenever
 237  *      a duplicate block is found.
 238  * The output of this function then goes to the output fd requested
 239  * by the caller of zfs_send().
 240  */
 241 static void *
 242 cksummer(void *arg)
 243 {
 244         dedup_arg_t *dda = arg;
 245         char *buf = zfs_alloc(dda->dedup_hdl, SPA_MAXBLOCKSIZE);
 246         dmu_replay_record_t thedrr = { 0 };
 247         dmu_replay_record_t *drr = &thedrr;
 248         FILE *ofp;
 249         int outfd;
 250         dedup_table_t ddt;
 251         zio_cksum_t stream_cksum;
 252         uint64_t numbuckets;
 253
 254 #ifdef _ILP32
 255         ddt.max_ddt_size = SMALLEST_POSSIBLE_MAX_DDT_MB << 20;
 256 #else
 257         uint64_t physmem = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
 258         ddt.max_ddt_size =
 259             MAX((physmem * MAX_DDT_PHYSMEM_PERCENT) / 100,
 260             SMALLEST_POSSIBLE_MAX_DDT_MB << 20);
 261 #endif
 262
 263         numbuckets = ddt.max_ddt_size / (sizeof (dedup_entry_t));
 264
 265         /*
 266          * numbuckets must be a power of 2.  Increase number to
 267          * a power of 2 if necessary.
 268          */
 269         if (!ISP2(numbuckets))
 270                 numbuckets = 1ULL << high_order_bit(numbuckets);
 271
 272         ddt.dedup_hash_array = calloc(numbuckets, sizeof (dedup_entry_t *));
 273         ddt.ddecache = umem_cache_create("dde", sizeof (dedup_entry_t), 0,
 274             NULL, NULL, NULL, NULL, NULL, 0);
 275         ddt.cur_ddt_size = numbuckets * sizeof (dedup_entry_t *);
 276         ddt.numhashbits = high_order_bit(numbuckets) - 1;
 277         ddt.ddt_full = B_FALSE;
 278
 279         outfd = dda->outputfd;
 280         ofp = fdopen(dda->inputfd, "r");
 281         while (ssread(drr, sizeof (*drr), ofp) != 0) {
 282
 283                 /*
 284                  * kernel filled in checksum, we are going to write same
 285                  * record, but need to regenerate checksum.
 286                  */
 287                 if (drr->drr_type != DRR_BEGIN) {
 288                         bzero(&drr->drr_u.drr_checksum.drr_checksum,
 289                             sizeof (drr->drr_u.drr_checksum.drr_checksum));
 290                 }
 291
 292                 switch (drr->drr_type) {
 293                 case DRR_BEGIN:
 294                 {
 295                         struct drr_begin *drrb = &drr->drr_u.drr_begin;
 296                         int fflags;
 297                         int sz = 0;
 298                         ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0);
 299
 300                         ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC);
 301
 302                         /* set the DEDUP feature flag for this stream */
 303                         fflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
 304                         fflags |= (DMU_BACKUP_FEATURE_DEDUP |
 305                             DMU_BACKUP_FEATURE_DEDUPPROPS);
 306                         DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, fflags);
 307
 308                         if (drr->drr_payloadlen != 0) {
 309                                 sz = drr->drr_payloadlen;
 310
 311                                 if (sz > SPA_MAXBLOCKSIZE) {
 312                                         buf = zfs_realloc(dda->dedup_hdl, buf,
 313                                             SPA_MAXBLOCKSIZE, sz);
 314                                 }
 315                                 (void) ssread(buf, sz, ofp);
 316                                 if (ferror(stdin))
 317                                         perror("fread");
 318                         }
 319                         if (dump_record(drr, buf, sz, &stream_cksum,
 320                             outfd) != 0)
 321                                 goto out;
 322                         break;
 323                 }
 324
 325                 case DRR_END:
 326                 {
 327                         struct drr_end *drre = &drr->drr_u.drr_end;
 328                         /* use the recalculated checksum */
 329                         drre->drr_checksum = stream_cksum;
 330                         if (dump_record(drr, NULL, 0, &stream_cksum,
 331                             outfd) != 0)
 332                                 goto out;
 333                         break;
 334                 }
 335
 336                 case DRR_OBJECT:
 337                 {
 338                         struct drr_object *drro = &drr->drr_u.drr_object;
 339                         if (drro->drr_bonuslen > 0) {
 340                                 (void) ssread(buf,
 341                                     DRR_OBJECT_PAYLOAD_SIZE(drro), ofp);
 342                         }
 343                         if (dump_record(drr, buf, DRR_OBJECT_PAYLOAD_SIZE(drro),
 344                             &stream_cksum, outfd) != 0)
 345                                 goto out;
 346                         break;
 347                 }
 348
 349                 case DRR_SPILL:
 350                 {
 351                         struct drr_spill *drrs = &drr->drr_u.drr_spill;
 352                         (void) ssread(buf, DRR_SPILL_PAYLOAD_SIZE(drrs), ofp);
 353                         if (dump_record(drr, buf, DRR_SPILL_PAYLOAD_SIZE(drrs),
 354                             &stream_cksum, outfd) != 0)
 355                                 goto out;
 356                         break;
 357                 }
 358
 359                 case DRR_FREEOBJECTS:
 360                 {
 361                         if (dump_record(drr, NULL, 0, &stream_cksum,
 362                             outfd) != 0)
 363                                 goto out;
 364                         break;
 365                 }
 366
 367                 case DRR_WRITE:
 368                 {
 369                         struct drr_write *drrw = &drr->drr_u.drr_write;
 370                         dataref_t       dataref;
 371                         uint64_t        payload_size;
 372
 373                         payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw);
 374                         (void) ssread(buf, payload_size, ofp);
 375
 376                         /*
 377                          * Use the existing checksum if it's dedup-capable,
 378                          * else calculate a SHA256 checksum for it.
 379                          */
 380
 381                         if (ZIO_CHECKSUM_EQUAL(drrw->drr_key.ddk_cksum,
 382                             zero_cksum) ||
 383                             !DRR_IS_DEDUP_CAPABLE(drrw->drr_flags)) {
 384                                 SHA2_CTX ctx;
 385                                 zio_cksum_t tmpsha256;
 386
 387                                 SHA2Init(SHA256, &ctx);
 388                                 SHA2Update(&ctx, buf, payload_size);
 389                                 SHA2Final(&tmpsha256, &ctx);
 390
 391                                 drrw->drr_key.ddk_cksum.zc_word[0] =
 392                                     BE_64(tmpsha256.zc_word[0]);
 393                                 drrw->drr_key.ddk_cksum.zc_word[1] =
 394                                     BE_64(tmpsha256.zc_word[1]);
 395                                 drrw->drr_key.ddk_cksum.zc_word[2] =
 396                                     BE_64(tmpsha256.zc_word[2]);
 397                                 drrw->drr_key.ddk_cksum.zc_word[3] =
 398                                     BE_64(tmpsha256.zc_word[3]);
 399                                 drrw->drr_checksumtype = ZIO_CHECKSUM_SHA256;
 400                                 drrw->drr_flags |= DRR_CHECKSUM_DEDUP;
 401                         }
 402
 403                         dataref.ref_guid = drrw->drr_toguid;
 404                         dataref.ref_object = drrw->drr_object;
 405                         dataref.ref_offset = drrw->drr_offset;
 406
 407                         if (ddt_update(dda->dedup_hdl, &ddt,
 408                             &drrw->drr_key.ddk_cksum, drrw->drr_key.ddk_prop,
 409                             &dataref)) {
 410                                 dmu_replay_record_t wbr_drr = {0};
 411                                 struct drr_write_byref *wbr_drrr =
 412                                     &wbr_drr.drr_u.drr_write_byref;
 413
 414                                 /* block already present in stream */
 415                                 wbr_drr.drr_type = DRR_WRITE_BYREF;
 416
 417                                 wbr_drrr->drr_object = drrw->drr_object;
 418                                 wbr_drrr->drr_offset = drrw->drr_offset;
 419                                 wbr_drrr->drr_length = drrw->drr_logical_size;
 420                                 wbr_drrr->drr_toguid = drrw->drr_toguid;
 421                                 wbr_drrr->drr_refguid = dataref.ref_guid;
 422                                 wbr_drrr->drr_refobject =
 423                                     dataref.ref_object;
 424                                 wbr_drrr->drr_refoffset =
 425                                     dataref.ref_offset;
 426
 427                                 wbr_drrr->drr_checksumtype =
 428                                     drrw->drr_checksumtype;
 429                                 wbr_drrr->drr_flags = drrw->drr_flags;
 430                                 wbr_drrr->drr_key.ddk_cksum =
 431                                     drrw->drr_key.ddk_cksum;
 432                                 wbr_drrr->drr_key.ddk_prop =
 433                                     drrw->drr_key.ddk_prop;
 434
 435                                 if (dump_record(&wbr_drr, NULL, 0,
 436                                     &stream_cksum, outfd) != 0)
 437                                         goto out;
 438                         } else {
 439                                 /* block not previously seen */
 440                                 if (dump_record(drr, buf, payload_size,
 441                                     &stream_cksum, outfd) != 0)
 442                                         goto out;
 443                         }
 444                         break;
 445                 }
 446
 447                 case DRR_WRITE_EMBEDDED:
 448                 {
 449                         struct drr_write_embedded *drrwe =
 450                             &drr->drr_u.drr_write_embedded;
 451                         (void) ssread(buf,
 452                             P2ROUNDUP((uint64_t)drrwe->drr_psize, 8), ofp);
 453                         if (dump_record(drr, buf,
 454                             P2ROUNDUP((uint64_t)drrwe->drr_psize, 8),
 455                             &stream_cksum, outfd) != 0)
 456                                 goto out;
 457                         break;
 458                 }
 459
 460                 case DRR_FREE:
 461                 {
 462                         if (dump_record(drr, NULL, 0, &stream_cksum,
 463                             outfd) != 0)
 464                                 goto out;
 465                         break;
 466                 }
 467
 468                 case DRR_OBJECT_RANGE:
 469                 {
 470                         if (dump_record(drr, NULL, 0, &stream_cksum,
 471                             outfd) != 0)
 472                                 goto out;
 473                         break;
 474                 }
 475
 476                 default:
 477                         (void) fprintf(stderr, "INVALID record type 0x%x\n",
 478                             drr->drr_type);
 479                         /* should never happen, so assert */
 480                         assert(B_FALSE);
 481                 }
 482         }
 483 out:
 484         umem_cache_destroy(ddt.ddecache);
 485         free(ddt.dedup_hash_array);
 486         free(buf);
 487         (void) fclose(ofp);
 488
 489         return (NULL);
 490 }
 491
 492 /*
 493  * Routines for dealing with the AVL tree of fs-nvlists
 494  */
 495 typedef struct fsavl_node {
 496         avl_node_t fn_node;
 497         nvlist_t *fn_nvfs;
 498         char *fn_snapname;
 499         uint64_t fn_guid;
 500 } fsavl_node_t;
 501
 502 static int
 503 fsavl_compare(const void *arg1, const void *arg2)
 504 {
 505         const fsavl_node_t *fn1 = (const fsavl_node_t *)arg1;
 506         const fsavl_node_t *fn2 = (const fsavl_node_t *)arg2;
 507
 508         return (AVL_CMP(fn1->fn_guid, fn2->fn_guid));
 509 }
 510
 511 /*
 512  * Given the GUID of a snapshot, find its containing filesystem and
 513  * (optionally) name.
 514  */
 515 static nvlist_t *
 516 fsavl_find(avl_tree_t *avl, uint64_t snapguid, char **snapname)
 517 {
 518         fsavl_node_t fn_find;
 519         fsavl_node_t *fn;
 520
 521         fn_find.fn_guid = snapguid;
 522
 523         fn = avl_find(avl, &fn_find, NULL);
 524         if (fn) {
 525                 if (snapname)
 526                         *snapname = fn->fn_snapname;
 527                 return (fn->fn_nvfs);
 528         }
 529         return (NULL);
 530 }
 531
 532 static void
 533 fsavl_destroy(avl_tree_t *avl)
 534 {
 535         fsavl_node_t *fn;
 536         void *cookie;
 537
 538         if (avl == NULL)
 539                 return;
 540
 541         cookie = NULL;
 542         while ((fn = avl_destroy_nodes(avl, &cookie)) != NULL)
 543                 free(fn);
 544         avl_destroy(avl);
 545         free(avl);
 546 }
 547
 548 /*
 549  * Given an nvlist, produce an avl tree of snapshots, ordered by guid
 550  */
 551 static avl_tree_t *
 552 fsavl_create(nvlist_t *fss)
 553 {
 554         avl_tree_t *fsavl;
 555         nvpair_t *fselem = NULL;
 556
 557         if ((fsavl = malloc(sizeof (avl_tree_t))) == NULL)
 558                 return (NULL);
 559
 560         avl_create(fsavl, fsavl_compare, sizeof (fsavl_node_t),
 561             offsetof(fsavl_node_t, fn_node));
 562
 563         while ((fselem = nvlist_next_nvpair(fss, fselem)) != NULL) {
 564                 nvlist_t *nvfs, *snaps;
 565                 nvpair_t *snapelem = NULL;
 566
 567                 VERIFY(0 == nvpair_value_nvlist(fselem, &nvfs));
 568                 VERIFY(0 == nvlist_lookup_nvlist(nvfs, "snaps", &snaps));
 569
 570                 while ((snapelem =
 571                     nvlist_next_nvpair(snaps, snapelem)) != NULL) {
 572                         fsavl_node_t *fn;
 573                         uint64_t guid;
 574
 575                         VERIFY(0 == nvpair_value_uint64(snapelem, &guid));
 576                         if ((fn = malloc(sizeof (fsavl_node_t))) == NULL) {
 577                                 fsavl_destroy(fsavl);
 578                                 return (NULL);
 579                         }
 580                         fn->fn_nvfs = nvfs;
 581                         fn->fn_snapname = nvpair_name(snapelem);
 582                         fn->fn_guid = guid;
 583
 584                         /*
 585                          * Note: if there are multiple snaps with the
 586                          * same GUID, we ignore all but one.
 587                          */
 588                         if (avl_find(fsavl, fn, NULL) == NULL)
 589                                 avl_add(fsavl, fn);
 590                         else
 591                                 free(fn);
 592                 }
 593         }
 594
 595         return (fsavl);
 596 }
 597
 598 /*
 599  * Routines for dealing with the giant nvlist of fs-nvlists, etc.
 600  */
 601 typedef struct send_data {
 602         /*
 603          * assigned inside every recursive call,
 604          * restored from *_save on return:
 605          *
 606          * guid of fromsnap snapshot in parent dataset
 607          * txg of fromsnap snapshot in current dataset
 608          * txg of tosnap snapshot in current dataset
 609          */
 610
 611         uint64_t parent_fromsnap_guid;
 612         uint64_t fromsnap_txg;
 613         uint64_t tosnap_txg;
 614
 615         /* the nvlists get accumulated during depth-first traversal */
 616         nvlist_t *parent_snaps;
 617         nvlist_t *fss;
 618         nvlist_t *snapprops;
 619
 620         /* send-receive configuration, does not change during traversal */
 621         const char *fsname;
 622         const char *fromsnap;
 623         const char *tosnap;
 624         boolean_t raw;
 625         boolean_t backup;
 626         boolean_t recursive;
 627         boolean_t verbose;
 628         boolean_t seenfrom;
 629         boolean_t seento;
 630
 631         /*
 632          * The header nvlist is of the following format:
 633          * {
 634          *   "tosnap" -> string
 635          *   "fromsnap" -> string (if incremental)
 636          *   "fss" -> {
 637          *      id -> {
 638          *
 639          *       "name" -> string (full name; for debugging)
 640          *       "parentfromsnap" -> number (guid of fromsnap in parent)
 641          *
 642          *       "props" -> { name -> value (only if set here) }
 643          *       "snaps" -> { name (lastname) -> number (guid) }
 644          *       "snapprops" -> { name (lastname) -> { name -> value } }
 645          *
 646          *       "origin" -> number (guid) (if clone)
 647          *       "is_encroot" -> boolean
 648          *       "sent" -> boolean (not on-disk)
 649          *      }
 650          *   }
 651          * }
 652          *
 653          */
 654 } send_data_t;
 655
 656 static void
 657 send_iterate_prop(zfs_handle_t *zhp, boolean_t received_only, nvlist_t *nv);
 658
 659 static int
 660 send_iterate_snap(zfs_handle_t *zhp, void *arg)
 661 {
 662         send_data_t *sd = arg;
 663         uint64_t guid = zhp->zfs_dmustats.dds_guid;
 664         uint64_t txg = zhp->zfs_dmustats.dds_creation_txg;
 665         char *snapname;
 666         nvlist_t *nv;
 667         boolean_t isfromsnap, istosnap, istosnapwithnofrom;
 668
 669         snapname = strrchr(zhp->zfs_name, '@')+1;
 670         isfromsnap = (sd->fromsnap != NULL &&
 671             strcmp(sd->fromsnap, snapname) == 0);
 672         istosnap = (sd->tosnap != NULL && (strcmp(sd->tosnap, snapname) == 0));
 673         istosnapwithnofrom = (istosnap && sd->fromsnap == NULL);
 674
 675         if (sd->tosnap_txg != 0 && txg > sd->tosnap_txg) {
 676                 if (sd->verbose) {
 677                         (void) fprintf(stderr, dgettext(TEXT_DOMAIN,
 678                             "skipping snapshot %s because it was created "
 679                             "after the destination snapshot (%s)\n"),
 680                             zhp->zfs_name, sd->tosnap);
 681                 }
 682                 zfs_close(zhp);
 683                 return (0);
 684         }
 685
 686         VERIFY(0 == nvlist_add_uint64(sd->parent_snaps, snapname, guid));
 687         /*
 688          * NB: if there is no fromsnap here (it's a newly created fs in
 689          * an incremental replication), we will substitute the tosnap.
 690          */
 691         if (isfromsnap || (sd->parent_fromsnap_guid == 0 && istosnap)) {
 692                 sd->parent_fromsnap_guid = guid;
 693         }
 694
 695         if (!sd->recursive) {
 696                 if (!sd->seenfrom && isfromsnap) {
 697                         sd->seenfrom = B_TRUE;
 698                         zfs_close(zhp);
 699                         return (0);
 700                 }
 701
 702                 if ((sd->seento || !sd->seenfrom) && !istosnapwithnofrom) {
 703                         zfs_close(zhp);
 704                         return (0);
 705                 }
 706
 707                 if (istosnap)
 708                         sd->seento = B_TRUE;
 709         }
 710
 711         VERIFY(0 == nvlist_alloc(&nv, NV_UNIQUE_NAME, 0));
 712         send_iterate_prop(zhp, sd->backup, nv);
 713         VERIFY(0 == nvlist_add_nvlist(sd->snapprops, snapname, nv));
 714         nvlist_free(nv);
 715
 716         zfs_close(zhp);
 717         return (0);
 718 }
 719
 720 static void
 721 send_iterate_prop(zfs_handle_t *zhp, boolean_t received_only, nvlist_t *nv)
 722 {
 723         nvlist_t *props = NULL;
 724         nvpair_t *elem = NULL;
 725
 726         if (received_only)
 727                 props = zfs_get_recvd_props(zhp);
 728         else
 729                 props = zhp->zfs_props;
 730
 731         while ((elem = nvlist_next_nvpair(props, elem)) != NULL) {
 732                 char *propname = nvpair_name(elem);
 733                 zfs_prop_t prop = zfs_name_to_prop(propname);
 734                 nvlist_t *propnv;
 735
 736                 if (!zfs_prop_user(propname)) {
 737                         /*
 738                          * Realistically, this should never happen.  However,
 739                          * we want the ability to add DSL properties without
 740                          * needing to make incompatible version changes.  We
 741                          * need to ignore unknown properties to allow older
 742                          * software to still send datasets containing these
 743                          * properties, with the unknown properties elided.
 744                          */
 745                         if (prop == ZPROP_INVAL)
 746                                 continue;
 747
 748                         if (zfs_prop_readonly(prop))
 749                                 continue;
 750                 }
 751
 752                 verify(nvpair_value_nvlist(elem, &propnv) == 0);
 753                 if (prop == ZFS_PROP_QUOTA || prop == ZFS_PROP_RESERVATION ||
 754                     prop == ZFS_PROP_REFQUOTA ||
 755                     prop == ZFS_PROP_REFRESERVATION) {
 756                         char *source;
 757                         uint64_t value;
 758                         verify(nvlist_lookup_uint64(propnv,
 759                             ZPROP_VALUE, &value) == 0);
 760                         if (zhp->zfs_type == ZFS_TYPE_SNAPSHOT)
 761                                 continue;
 762                         /*
 763                          * May have no source before SPA_VERSION_RECVD_PROPS,
 764                          * but is still modifiable.
 765                          */
 766                         if (nvlist_lookup_string(propnv,
 767                             ZPROP_SOURCE, &source) == 0) {
 768                                 if ((strcmp(source, zhp->zfs_name) != 0) &&
 769                                     (strcmp(source,
 770                                     ZPROP_SOURCE_VAL_RECVD) != 0))
 771                                         continue;
 772                         }
 773                 } else {
 774                         char *source;
 775                         if (nvlist_lookup_string(propnv,
 776                             ZPROP_SOURCE, &source) != 0)
 777                                 continue;
 778                         if ((strcmp(source, zhp->zfs_name) != 0) &&
 779                             (strcmp(source, ZPROP_SOURCE_VAL_RECVD) != 0))
 780                                 continue;
 781                 }
 782
 783                 if (zfs_prop_user(propname) ||
 784                     zfs_prop_get_type(prop) == PROP_TYPE_STRING) {
 785                         char *value;
 786                         verify(nvlist_lookup_string(propnv,
 787                             ZPROP_VALUE, &value) == 0);
 788                         VERIFY(0 == nvlist_add_string(nv, propname, value));
 789                 } else {
 790                         uint64_t value;
 791                         verify(nvlist_lookup_uint64(propnv,
 792                             ZPROP_VALUE, &value) == 0);
 793                         VERIFY(0 == nvlist_add_uint64(nv, propname, value));
 794                 }
 795         }
 796 }
 797
 798 /*
 799  * returns snapshot creation txg
 800  * and returns 0 if the snapshot does not exist
 801  */
 802 static uint64_t
 803 get_snap_txg(libzfs_handle_t *hdl, const char *fs, const char *snap)
 804 {
 805         char name[ZFS_MAX_DATASET_NAME_LEN];
 806         uint64_t txg = 0;
 807
 808         if (fs == NULL || fs[0] == '\0' || snap == NULL || snap[0] == '\0')
 809                 return (txg);
 810
 811         (void) snprintf(name, sizeof (name), "%s@%s", fs, snap);
 812         if (zfs_dataset_exists(hdl, name, ZFS_TYPE_SNAPSHOT)) {
 813                 zfs_handle_t *zhp = zfs_open(hdl, name, ZFS_TYPE_SNAPSHOT);
 814                 if (zhp != NULL) {
 815                         txg = zfs_prop_get_int(zhp, ZFS_PROP_CREATETXG);
 816                         zfs_close(zhp);
 817                 }
 818         }
 819
 820         return (txg);
 821 }
 822
 823 /*
 824  * recursively generate nvlists describing datasets.  See comment
 825  * for the data structure send_data_t above for description of contents
 826  * of the nvlist.
 827  */
 828 static int
 829 send_iterate_fs(zfs_handle_t *zhp, void *arg)
 830 {
 831         send_data_t *sd = arg;
 832         nvlist_t *nvfs = NULL, *nv = NULL;
 833         int rv = 0;
 834         uint64_t parent_fromsnap_guid_save = sd->parent_fromsnap_guid;
 835         uint64_t fromsnap_txg_save = sd->fromsnap_txg;
 836         uint64_t tosnap_txg_save = sd->tosnap_txg;
 837         uint64_t txg = zhp->zfs_dmustats.dds_creation_txg;
 838         uint64_t guid = zhp->zfs_dmustats.dds_guid;
 839         uint64_t fromsnap_txg, tosnap_txg;
 840         char guidstring[64];
 841
 842         fromsnap_txg = get_snap_txg(zhp->zfs_hdl, zhp->zfs_name, sd->fromsnap);
 843         if (fromsnap_txg != 0)
 844                 sd->fromsnap_txg = fromsnap_txg;
 845
 846         tosnap_txg = get_snap_txg(zhp->zfs_hdl, zhp->zfs_name, sd->tosnap);
 847         if (tosnap_txg != 0)
 848                 sd->tosnap_txg = tosnap_txg;
 849
 850         /*
 851          * on the send side, if the current dataset does not have tosnap,
 852          * perform two additional checks:
 853          *
 854          * - skip sending the current dataset if it was created later than
 855          *   the parent tosnap
 856          * - return error if the current dataset was created earlier than
 857          *   the parent tosnap
 858          */
 859         if (sd->tosnap != NULL && tosnap_txg == 0) {
 860                 if (sd->tosnap_txg != 0 && txg > sd->tosnap_txg) {
 861                         if (sd->verbose) {
 862                                 (void) fprintf(stderr, dgettext(TEXT_DOMAIN,
 863                                     "skipping dataset %s: snapshot %s does "
 864                                     "not exist\n"), zhp->zfs_name, sd->tosnap);
 865                         }
 866                 } else {
 867                         (void) fprintf(stderr, dgettext(TEXT_DOMAIN,
 868                             "cannot send %s@%s%s: snapshot %s@%s does not "
 869                             "exist\n"), sd->fsname, sd->tosnap, sd->recursive ?
 870                             dgettext(TEXT_DOMAIN, " recursively") : "",
 871                             zhp->zfs_name, sd->tosnap);
 872                         rv = -1;
 873                 }
 874                 goto out;
 875         }
 876
 877         VERIFY(0 == nvlist_alloc(&nvfs, NV_UNIQUE_NAME, 0));
 878         VERIFY(0 == nvlist_add_string(nvfs, "name", zhp->zfs_name));
 879         VERIFY(0 == nvlist_add_uint64(nvfs, "parentfromsnap",
 880             sd->parent_fromsnap_guid));
 881
 882         if (zhp->zfs_dmustats.dds_origin[0]) {
 883                 zfs_handle_t *origin = zfs_open(zhp->zfs_hdl,
 884                     zhp->zfs_dmustats.dds_origin, ZFS_TYPE_SNAPSHOT);
 885                 if (origin == NULL) {
 886                         rv = -1;
 887                         goto out;
 888                 }
 889                 VERIFY(0 == nvlist_add_uint64(nvfs, "origin",
 890                     origin->zfs_dmustats.dds_guid));
 891
 892                 zfs_close(origin);
 893         }
 894
 895         /* iterate over props */
 896         VERIFY(0 == nvlist_alloc(&nv, NV_UNIQUE_NAME, 0));
 897         send_iterate_prop(zhp, sd->backup, nv);
 898
 899         if (zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION) != ZIO_CRYPT_OFF) {
 900                 boolean_t encroot;
 901
 902                 /* determine if this dataset is an encryption root */
 903                 if (zfs_crypto_get_encryption_root(zhp, &encroot, NULL) != 0) {
 904                         rv = -1;
 905                         goto out;
 906                 }
 907
 908                 if (encroot)
 909                         VERIFY(0 == nvlist_add_boolean(nvfs, "is_encroot"));
 910
 911                 /*
 912                  * Encrypted datasets can only be sent with properties if
 913                  * the raw flag is specified because the receive side doesn't
 914                  * currently have a mechanism for recursively asking the user
 915                  * for new encryption parameters.
 916                  */
 917                 if (!sd->raw) {
 918                         (void) fprintf(stderr, dgettext(TEXT_DOMAIN,
 919                             "cannot send %s@%s: encrypted dataset %s may not "
 920                             "be sent with properties without the raw flag\n"),
 921                             sd->fsname, sd->tosnap, zhp->zfs_name);
 922                         rv = -1;
 923                         goto out;
 924                 }
 925
 926         }
 927
 928         VERIFY(0 == nvlist_add_nvlist(nvfs, "props", nv));
 929
 930         /* iterate over snaps, and set sd->parent_fromsnap_guid */
 931         sd->parent_fromsnap_guid = 0;
 932         VERIFY(0 == nvlist_alloc(&sd->parent_snaps, NV_UNIQUE_NAME, 0));
 933         VERIFY(0 == nvlist_alloc(&sd->snapprops, NV_UNIQUE_NAME, 0));
 934         (void) zfs_iter_snapshots_sorted(zhp, send_iterate_snap, sd);
 935         VERIFY(0 == nvlist_add_nvlist(nvfs, "snaps", sd->parent_snaps));
 936         VERIFY(0 == nvlist_add_nvlist(nvfs, "snapprops", sd->snapprops));
 937         nvlist_free(sd->parent_snaps);
 938         nvlist_free(sd->snapprops);
 939
 940         /* add this fs to nvlist */
 941         (void) snprintf(guidstring, sizeof (guidstring),
 942             "0x%llx", (longlong_t)guid);
 943         VERIFY(0 == nvlist_add_nvlist(sd->fss, guidstring, nvfs));
 944
 945         /* iterate over children */
 946         if (sd->recursive)
 947                 rv = zfs_iter_filesystems(zhp, send_iterate_fs, sd);
 948
 949 out:
 950         sd->parent_fromsnap_guid = parent_fromsnap_guid_save;
 951         sd->fromsnap_txg = fromsnap_txg_save;
 952         sd->tosnap_txg = tosnap_txg_save;
 953         nvlist_free(nv);
 954         nvlist_free(nvfs);
 955
 956         zfs_close(zhp);
 957         return (rv);
 958 }
 959
 960 static int
 961 gather_nvlist(libzfs_handle_t *hdl, const char *fsname, const char *fromsnap,
 962     const char *tosnap, boolean_t recursive, boolean_t raw, boolean_t verbose,
 963     boolean_t backup, nvlist_t **nvlp, avl_tree_t **avlp)
 964 {
 965         zfs_handle_t *zhp;
 966         send_data_t sd = { 0 };
 967         int error;
 968
 969         zhp = zfs_open(hdl, fsname, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
 970         if (zhp == NULL)
 971                 return (EZFS_BADTYPE);
 972
 973         VERIFY(0 == nvlist_alloc(&sd.fss, NV_UNIQUE_NAME, 0));
 974         sd.fsname = fsname;
 975         sd.fromsnap = fromsnap;
 976         sd.tosnap = tosnap;
 977         sd.recursive = recursive;
 978         sd.raw = raw;
 979         sd.verbose = verbose;
 980         sd.backup = backup;
 981
 982         if ((error = send_iterate_fs(zhp, &sd)) != 0) {
 983                 nvlist_free(sd.fss);
 984                 if (avlp != NULL)
 985                         *avlp = NULL;
 986                 *nvlp = NULL;
 987                 return (error);
 988         }
 989
 990         if (avlp != NULL && (*avlp = fsavl_create(sd.fss)) == NULL) {
 991                 nvlist_free(sd.fss);
 992                 *nvlp = NULL;
 993                 return (EZFS_NOMEM);
 994         }
 995
 996         *nvlp = sd.fss;
 997         return (0);
 998 }
 999
1000 /*
1001  * Routines specific to "zfs send"
1002  */
1003 typedef struct send_dump_data {
1004         /* these are all just the short snapname (the part after the @) */
1005         const char *fromsnap;
1006         const char *tosnap;
1007         char prevsnap[ZFS_MAX_DATASET_NAME_LEN];
1008         uint64_t prevsnap_obj;
1009         boolean_t seenfrom, seento, replicate, doall, fromorigin;
1010         boolean_t verbose, dryrun, parsable, progress, embed_data, std_out;
1011         boolean_t large_block, compress, raw;
1012         int outfd;
1013         boolean_t err;
1014         nvlist_t *fss;
1015         nvlist_t *snapholds;
1016         avl_tree_t *fsavl;
1017         snapfilter_cb_t *filter_cb;
1018         void *filter_cb_arg;
1019         nvlist_t *debugnv;
1020         char holdtag[ZFS_MAX_DATASET_NAME_LEN];
1021         int cleanup_fd;
1022         uint64_t size;
1023 } send_dump_data_t;
1024
1025 static int
1026 zfs_send_space(zfs_handle_t *zhp, const char *snapname, const char *from,
1027     enum lzc_send_flags flags, uint64_t *spacep)
1028 {
1029         libzfs_handle_t *hdl = zhp->zfs_hdl;
1030         int error;
1031
1032         assert(snapname != NULL);
1033         error = lzc_send_space(snapname, from, flags, spacep);
1034
1035         if (error != 0) {
1036                 char errbuf[1024];
1037                 (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
1038                     "warning: cannot estimate space for '%s'"), snapname);
1039
1040                 switch (error) {
1041                 case EXDEV:
1042                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1043                             "not an earlier snapshot from the same fs"));
1044                         return (zfs_error(hdl, EZFS_CROSSTARGET, errbuf));
1045
1046                 case ENOENT:
1047                         if (zfs_dataset_exists(hdl, snapname,
1048                             ZFS_TYPE_SNAPSHOT)) {
1049                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1050                                     "incremental source (%s) does not exist"),
1051                                     snapname);
1052                         }
1053                         return (zfs_error(hdl, EZFS_NOENT, errbuf));
1054
1055                 case EDQUOT:
1056                 case EFBIG:
1057                 case EIO:
1058                 case ENOLINK:
1059                 case ENOSPC:
1060                 case ENOSTR:
1061                 case ENXIO:
1062                 case EPIPE:
1063                 case ERANGE:
1064                 case EFAULT:
1065                 case EROFS:
1066                 case EINVAL:
1067                         zfs_error_aux(hdl, strerror(error));
1068                         return (zfs_error(hdl, EZFS_BADBACKUP, errbuf));
1069
1070                 default:
1071                         return (zfs_standard_error(hdl, error, errbuf));
1072                 }
1073         }
1074
1075         return (0);
1076 }
1077
1078 /*
1079  * Dumps a backup of the given snapshot (incremental from fromsnap if it's not
1080  * NULL) to the file descriptor specified by outfd.
1081  */
1082 static int
1083 dump_ioctl(zfs_handle_t *zhp, const char *fromsnap, uint64_t fromsnap_obj,
1084     boolean_t fromorigin, int outfd, enum lzc_send_flags flags,
1085     nvlist_t *debugnv)
1086 {
1087         zfs_cmd_t zc = {"\0"};
1088         libzfs_handle_t *hdl = zhp->zfs_hdl;
1089         nvlist_t *thisdbg;
1090
1091         assert(zhp->zfs_type == ZFS_TYPE_SNAPSHOT);
1092         assert(fromsnap_obj == 0 || !fromorigin);
1093
1094         (void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name));
1095         zc.zc_cookie = outfd;
1096         zc.zc_obj = fromorigin;
1097         zc.zc_sendobj = zfs_prop_get_int(zhp, ZFS_PROP_OBJSETID);
1098         zc.zc_fromobj = fromsnap_obj;
1099         zc.zc_flags = flags;
1100
1101         VERIFY(0 == nvlist_alloc(&thisdbg, NV_UNIQUE_NAME, 0));
1102         if (fromsnap && fromsnap[0] != '\0') {
1103                 VERIFY(0 == nvlist_add_string(thisdbg,
1104                     "fromsnap", fromsnap));
1105         }
1106
1107         if (zfs_ioctl(zhp->zfs_hdl, ZFS_IOC_SEND, &zc) != 0) {
1108                 char errbuf[1024];
1109                 (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
1110                     "warning: cannot send '%s'"), zhp->zfs_name);
1111
1112                 VERIFY(0 == nvlist_add_uint64(thisdbg, "error", errno));
1113                 if (debugnv) {
1114                         VERIFY(0 == nvlist_add_nvlist(debugnv,
1115                             zhp->zfs_name, thisdbg));
1116                 }
1117                 nvlist_free(thisdbg);
1118
1119                 switch (errno) {
1120                 case EXDEV:
1121                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1122                             "not an earlier snapshot from the same fs"));
1123                         return (zfs_error(hdl, EZFS_CROSSTARGET, errbuf));
1124
1125                 case EACCES:
1126                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1127                             "source key must be loaded"));
1128                         return (zfs_error(hdl, EZFS_CRYPTOFAILED, errbuf));
1129
1130                 case ENOENT:
1131                         if (zfs_dataset_exists(hdl, zc.zc_name,
1132                             ZFS_TYPE_SNAPSHOT)) {
1133                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1134                                     "incremental source (@%s) does not exist"),
1135                                     zc.zc_value);
1136                         }
1137                         return (zfs_error(hdl, EZFS_NOENT, errbuf));
1138
1139                 case EDQUOT:
1140                 case EFBIG:
1141                 case EIO:
1142                 case ENOLINK:
1143                 case ENOSPC:
1144                 case ENOSTR:
1145                 case ENXIO:
1146                 case EPIPE:
1147                 case ERANGE:
1148                 case EFAULT:
1149                 case EROFS:
1150                         zfs_error_aux(hdl, strerror(errno));
1151                         return (zfs_error(hdl, EZFS_BADBACKUP, errbuf));
1152
1153                 default:
1154                         return (zfs_standard_error(hdl, errno, errbuf));
1155                 }
1156         }
1157
1158         if (debugnv)
1159                 VERIFY(0 == nvlist_add_nvlist(debugnv, zhp->zfs_name, thisdbg));
1160         nvlist_free(thisdbg);
1161
1162         return (0);
1163 }
1164
1165 static void
1166 gather_holds(zfs_handle_t *zhp, send_dump_data_t *sdd)
1167 {
1168         assert(zhp->zfs_type == ZFS_TYPE_SNAPSHOT);
1169
1170         /*
1171          * zfs_send() only sets snapholds for sends that need them,
1172          * e.g. replication and doall.
1173          */
1174         if (sdd->snapholds == NULL)
1175                 return;
1176
1177         fnvlist_add_string(sdd->snapholds, zhp->zfs_name, sdd->holdtag);
1178 }
1179
1180 static void *
1181 send_progress_thread(void *arg)
1182 {
1183         progress_arg_t *pa = arg;
1184         zfs_cmd_t zc = {"\0"};
1185         zfs_handle_t *zhp = pa->pa_zhp;
1186         libzfs_handle_t *hdl = zhp->zfs_hdl;
1187         unsigned long long bytes;
1188         char buf[16];
1189         time_t t;
1190         struct tm *tm;
1191
1192         (void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name));
1193
1194         if (!pa->pa_parsable)
1195                 (void) fprintf(stderr, "TIME        SENT   SNAPSHOT\n");
1196
1197         /*
1198          * Print the progress from ZFS_IOC_SEND_PROGRESS every second.
1199          */
1200         for (;;) {
1201                 (void) sleep(1);
1202
1203                 zc.zc_cookie = pa->pa_fd;
1204                 if (zfs_ioctl(hdl, ZFS_IOC_SEND_PROGRESS, &zc) != 0)
1205                         return ((void *)-1);
1206
1207                 (void) time(&t);
1208                 tm = localtime(&t);
1209                 bytes = zc.zc_cookie;
1210
1211                 if (pa->pa_parsable) {
1212                         (void) fprintf(stderr, "%02d:%02d:%02d\t%llu\t%s\n",
1213                             tm->tm_hour, tm->tm_min, tm->tm_sec,
1214                             bytes, zhp->zfs_name);
1215                 } else {
1216                         zfs_nicebytes(bytes, buf, sizeof (buf));
1217                         (void) fprintf(stderr, "%02d:%02d:%02d   %5s   %s\n",
1218                             tm->tm_hour, tm->tm_min, tm->tm_sec,
1219                             buf, zhp->zfs_name);
1220                 }
1221         }
1222 }
1223
1224 static void
1225 send_print_verbose(FILE *fout, const char *tosnap, const char *fromsnap,
1226     uint64_t size, boolean_t parsable)
1227 {
1228         if (parsable) {
1229                 if (fromsnap != NULL) {
1230                         (void) fprintf(fout, "incremental\t%s\t%s",
1231                             fromsnap, tosnap);
1232                 } else {
1233                         (void) fprintf(fout, "full\t%s",
1234                             tosnap);
1235                 }
1236         } else {
1237                 if (fromsnap != NULL) {
1238                         if (strchr(fromsnap, '@') == NULL &&
1239                             strchr(fromsnap, '#') == NULL) {
1240                                 (void) fprintf(fout, dgettext(TEXT_DOMAIN,
1241                                     "send from @%s to %s"),
1242                                     fromsnap, tosnap);
1243                         } else {
1244                                 (void) fprintf(fout, dgettext(TEXT_DOMAIN,
1245                                     "send from %s to %s"),
1246                                     fromsnap, tosnap);
1247                         }
1248                 } else {
1249                         (void) fprintf(fout, dgettext(TEXT_DOMAIN,
1250                             "full send of %s"),
1251                             tosnap);
1252                 }
1253         }
1254
1255         if (parsable) {
1256                 (void) fprintf(fout, "\t%llu",
1257                     (longlong_t)size);
1258         } else if (size != 0) {
1259                 char buf[16];
1260                 zfs_nicebytes(size, buf, sizeof (buf));
1261                 (void) fprintf(fout, dgettext(TEXT_DOMAIN,
1262                     " estimated size is %s"), buf);
1263         }
1264         (void) fprintf(fout, "\n");
1265 }
1266
1267 static int
1268 dump_snapshot(zfs_handle_t *zhp, void *arg)
1269 {
1270         send_dump_data_t *sdd = arg;
1271         progress_arg_t pa = { 0 };
1272         pthread_t tid;
1273         char *thissnap;
1274         enum lzc_send_flags flags = 0;
1275         int err;
1276         boolean_t isfromsnap, istosnap, fromorigin;
1277         boolean_t exclude = B_FALSE;
1278         FILE *fout = sdd->std_out ? stdout : stderr;
1279
1280         err = 0;
1281         thissnap = strchr(zhp->zfs_name, '@') + 1;
1282         isfromsnap = (sdd->fromsnap != NULL &&
1283             strcmp(sdd->fromsnap, thissnap) == 0);
1284
1285         if (!sdd->seenfrom && isfromsnap) {
1286                 gather_holds(zhp, sdd);
1287                 sdd->seenfrom = B_TRUE;
1288                 (void) strlcpy(sdd->prevsnap, thissnap,
1289                     sizeof (sdd->prevsnap));
1290                 sdd->prevsnap_obj = zfs_prop_get_int(zhp, ZFS_PROP_OBJSETID);
1291                 zfs_close(zhp);
1292                 return (0);
1293         }
1294
1295         if (sdd->seento || !sdd->seenfrom) {
1296                 zfs_close(zhp);
1297                 return (0);
1298         }
1299
1300         istosnap = (strcmp(sdd->tosnap, thissnap) == 0);
1301         if (istosnap)
1302                 sdd->seento = B_TRUE;
1303
1304         if (sdd->large_block)
1305                 flags |= LZC_SEND_FLAG_LARGE_BLOCK;
1306         if (sdd->embed_data)
1307                 flags |= LZC_SEND_FLAG_EMBED_DATA;
1308         if (sdd->compress)
1309                 flags |= LZC_SEND_FLAG_COMPRESS;
1310         if (sdd->raw)
1311                 flags |= LZC_SEND_FLAG_RAW;
1312
1313         if (!sdd->doall && !isfromsnap && !istosnap) {
1314                 if (sdd->replicate) {
1315                         char *snapname;
1316                         nvlist_t *snapprops;
1317                         /*
1318                          * Filter out all intermediate snapshots except origin
1319                          * snapshots needed to replicate clones.
1320                          */
1321                         nvlist_t *nvfs = fsavl_find(sdd->fsavl,
1322                             zhp->zfs_dmustats.dds_guid, &snapname);
1323
1324                         VERIFY(0 == nvlist_lookup_nvlist(nvfs,
1325                             "snapprops", &snapprops));
1326                         VERIFY(0 == nvlist_lookup_nvlist(snapprops,
1327                             thissnap, &snapprops));
1328                         exclude = !nvlist_exists(snapprops, "is_clone_origin");
1329                 } else {
1330                         exclude = B_TRUE;
1331                 }
1332         }
1333
1334         /*
1335          * If a filter function exists, call it to determine whether
1336          * this snapshot will be sent.
1337          */
1338         if (exclude || (sdd->filter_cb != NULL &&
1339             sdd->filter_cb(zhp, sdd->filter_cb_arg) == B_FALSE)) {
1340                 /*
1341                  * This snapshot is filtered out.  Don't send it, and don't
1342                  * set prevsnap_obj, so it will be as if this snapshot didn't
1343                  * exist, and the next accepted snapshot will be sent as
1344                  * an incremental from the last accepted one, or as the
1345                  * first (and full) snapshot in the case of a replication,
1346                  * non-incremental send.
1347                  */
1348                 zfs_close(zhp);
1349                 return (0);
1350         }
1351
1352         gather_holds(zhp, sdd);
1353         fromorigin = sdd->prevsnap[0] == '\0' &&
1354             (sdd->fromorigin || sdd->replicate);
1355
1356         if (sdd->verbose) {
1357                 uint64_t size = 0;
1358                 char fromds[ZFS_MAX_DATASET_NAME_LEN];
1359
1360                 if (sdd->prevsnap[0] != '\0') {
1361                         (void) strlcpy(fromds, zhp->zfs_name, sizeof (fromds));
1362                         *(strchr(fromds, '@') + 1) = '\0';
1363                         (void) strlcat(fromds, sdd->prevsnap, sizeof (fromds));
1364                 }
1365                 if (zfs_send_space(zhp, zhp->zfs_name,
1366                     sdd->prevsnap[0] ? fromds : NULL, flags, &size) != 0) {
1367                         size = 0; /* cannot estimate send space */
1368                 } else {
1369                         send_print_verbose(fout, zhp->zfs_name,
1370                             sdd->prevsnap[0] ? sdd->prevsnap : NULL,
1371                             size, sdd->parsable);
1372                 }
1373                 sdd->size += size;
1374         }
1375
1376         if (!sdd->dryrun) {
1377                 /*
1378                  * If progress reporting is requested, spawn a new thread to
1379                  * poll ZFS_IOC_SEND_PROGRESS at a regular interval.
1380                  */
1381                 if (sdd->progress) {
1382                         pa.pa_zhp = zhp;
1383                         pa.pa_fd = sdd->outfd;
1384                         pa.pa_parsable = sdd->parsable;
1385
1386                         if ((err = pthread_create(&tid, NULL,
1387                             send_progress_thread, &pa)) != 0) {
1388                                 zfs_close(zhp);
1389                                 return (err);
1390                         }
1391                 }
1392
1393                 err = dump_ioctl(zhp, sdd->prevsnap, sdd->prevsnap_obj,
1394                     fromorigin, sdd->outfd, flags, sdd->debugnv);
1395
1396                 if (sdd->progress) {
1397                         (void) pthread_cancel(tid);
1398                         (void) pthread_join(tid, NULL);
1399                 }
1400         }
1401
1402         (void) strcpy(sdd->prevsnap, thissnap);
1403         sdd->prevsnap_obj = zfs_prop_get_int(zhp, ZFS_PROP_OBJSETID);
1404         zfs_close(zhp);
1405         return (err);
1406 }
1407
1408 static int
1409 dump_filesystem(zfs_handle_t *zhp, void *arg)
1410 {
1411         int rv = 0;
1412         send_dump_data_t *sdd = arg;
1413         boolean_t missingfrom = B_FALSE;
1414         zfs_cmd_t zc = {"\0"};
1415
1416         (void) snprintf(zc.zc_name, sizeof (zc.zc_name), "%s@%s",
1417             zhp->zfs_name, sdd->tosnap);
1418         if (ioctl(zhp->zfs_hdl->libzfs_fd, ZFS_IOC_OBJSET_STATS, &zc) != 0) {
1419                 (void) fprintf(stderr, dgettext(TEXT_DOMAIN,
1420                     "WARNING: could not send %s@%s: does not exist\n"),
1421                     zhp->zfs_name, sdd->tosnap);
1422                 sdd->err = B_TRUE;
1423                 return (0);
1424         }
1425
1426         if (sdd->replicate && sdd->fromsnap) {
1427                 /*
1428                  * If this fs does not have fromsnap, and we're doing
1429                  * recursive, we need to send a full stream from the
1430                  * beginning (or an incremental from the origin if this
1431                  * is a clone).  If we're doing non-recursive, then let
1432                  * them get the error.
1433                  */
1434                 (void) snprintf(zc.zc_name, sizeof (zc.zc_name), "%s@%s",
1435                     zhp->zfs_name, sdd->fromsnap);
1436                 if (ioctl(zhp->zfs_hdl->libzfs_fd,
1437                     ZFS_IOC_OBJSET_STATS, &zc) != 0) {
1438                         missingfrom = B_TRUE;
1439                 }
1440         }
1441
1442         sdd->seenfrom = sdd->seento = sdd->prevsnap[0] = 0;
1443         sdd->prevsnap_obj = 0;
1444         if (sdd->fromsnap == NULL || missingfrom)
1445                 sdd->seenfrom = B_TRUE;
1446
1447         rv = zfs_iter_snapshots_sorted(zhp, dump_snapshot, arg);
1448         if (!sdd->seenfrom) {
1449                 (void) fprintf(stderr, dgettext(TEXT_DOMAIN,
1450                     "WARNING: could not send %s@%s:\n"
1451                     "incremental source (%s@%s) does not exist\n"),
1452                     zhp->zfs_name, sdd->tosnap,
1453                     zhp->zfs_name, sdd->fromsnap);
1454                 sdd->err = B_TRUE;
1455         } else if (!sdd->seento) {
1456                 if (sdd->fromsnap) {
1457                         (void) fprintf(stderr, dgettext(TEXT_DOMAIN,
1458                             "WARNING: could not send %s@%s:\n"
1459                             "incremental source (%s@%s) "
1460                             "is not earlier than it\n"),
1461                             zhp->zfs_name, sdd->tosnap,
1462                             zhp->zfs_name, sdd->fromsnap);
1463                 } else {
1464                         (void) fprintf(stderr, dgettext(TEXT_DOMAIN,
1465                             "WARNING: "
1466                             "could not send %s@%s: does not exist\n"),
1467                             zhp->zfs_name, sdd->tosnap);
1468                 }
1469                 sdd->err = B_TRUE;
1470         }
1471
1472         return (rv);
1473 }
1474
1475 static int
1476 dump_filesystems(zfs_handle_t *rzhp, void *arg)
1477 {
1478         send_dump_data_t *sdd = arg;
1479         nvpair_t *fspair;
1480         boolean_t needagain, progress;
1481
1482         if (!sdd->replicate)
1483                 return (dump_filesystem(rzhp, sdd));
1484
1485         /* Mark the clone origin snapshots. */
1486         for (fspair = nvlist_next_nvpair(sdd->fss, NULL); fspair;
1487             fspair = nvlist_next_nvpair(sdd->fss, fspair)) {
1488                 nvlist_t *nvfs;
1489                 uint64_t origin_guid = 0;
1490
1491                 VERIFY(0 == nvpair_value_nvlist(fspair, &nvfs));
1492                 (void) nvlist_lookup_uint64(nvfs, "origin", &origin_guid);
1493                 if (origin_guid != 0) {
1494                         char *snapname;
1495                         nvlist_t *origin_nv = fsavl_find(sdd->fsavl,
1496                             origin_guid, &snapname);
1497                         if (origin_nv != NULL) {
1498                                 nvlist_t *snapprops;
1499                                 VERIFY(0 == nvlist_lookup_nvlist(origin_nv,
1500                                     "snapprops", &snapprops));
1501                                 VERIFY(0 == nvlist_lookup_nvlist(snapprops,
1502                                     snapname, &snapprops));
1503                                 VERIFY(0 == nvlist_add_boolean(
1504                                     snapprops, "is_clone_origin"));
1505                         }
1506                 }
1507         }
1508 again:
1509         needagain = progress = B_FALSE;
1510         for (fspair = nvlist_next_nvpair(sdd->fss, NULL); fspair;
1511             fspair = nvlist_next_nvpair(sdd->fss, fspair)) {
1512                 nvlist_t *fslist, *parent_nv;
1513                 char *fsname;
1514                 zfs_handle_t *zhp;
1515                 int err;
1516                 uint64_t origin_guid = 0;
1517                 uint64_t parent_guid = 0;
1518
1519                 VERIFY(nvpair_value_nvlist(fspair, &fslist) == 0);
1520                 if (nvlist_lookup_boolean(fslist, "sent") == 0)
1521                         continue;
1522
1523                 VERIFY(nvlist_lookup_string(fslist, "name", &fsname) == 0);
1524                 (void) nvlist_lookup_uint64(fslist, "origin", &origin_guid);
1525                 (void) nvlist_lookup_uint64(fslist, "parentfromsnap",
1526                     &parent_guid);
1527
1528                 if (parent_guid != 0) {
1529                         parent_nv = fsavl_find(sdd->fsavl, parent_guid, NULL);
1530                         if (!nvlist_exists(parent_nv, "sent")) {
1531                                 /* parent has not been sent; skip this one */
1532                                 needagain = B_TRUE;
1533                                 continue;
1534                         }
1535                 }
1536
1537                 if (origin_guid != 0) {
1538                         nvlist_t *origin_nv = fsavl_find(sdd->fsavl,
1539                             origin_guid, NULL);
1540                         if (origin_nv != NULL &&
1541                             !nvlist_exists(origin_nv, "sent")) {
1542                                 /*
1543                                  * origin has not been sent yet;
1544                                  * skip this clone.
1545                                  */
1546                                 needagain = B_TRUE;
1547                                 continue;
1548                         }
1549                 }
1550
1551                 zhp = zfs_open(rzhp->zfs_hdl, fsname, ZFS_TYPE_DATASET);
1552                 if (zhp == NULL)
1553                         return (-1);
1554                 err = dump_filesystem(zhp, sdd);
1555                 VERIFY(nvlist_add_boolean(fslist, "sent") == 0);
1556                 progress = B_TRUE;
1557                 zfs_close(zhp);
1558                 if (err)
1559                         return (err);
1560         }
1561         if (needagain) {
1562                 assert(progress);
1563                 goto again;
1564         }
1565
1566         /* clean out the sent flags in case we reuse this fss */
1567         for (fspair = nvlist_next_nvpair(sdd->fss, NULL); fspair;
1568             fspair = nvlist_next_nvpair(sdd->fss, fspair)) {
1569                 nvlist_t *fslist;
1570
1571                 VERIFY(nvpair_value_nvlist(fspair, &fslist) == 0);
1572                 (void) nvlist_remove_all(fslist, "sent");
1573         }
1574
1575         return (0);
1576 }
1577
1578 nvlist_t *
1579 zfs_send_resume_token_to_nvlist(libzfs_handle_t *hdl, const char *token)
1580 {
1581         unsigned int version;
1582         int nread, i;
1583         unsigned long long checksum, packed_len;
1584
1585         /*
1586          * Decode token header, which is:
1587          *   <token version>-<checksum of payload>-<uncompressed payload length>
1588          * Note that the only supported token version is 1.
1589          */
1590         nread = sscanf(token, "%u-%llx-%llx-",
1591             &version, &checksum, &packed_len);
1592         if (nread != 3) {
1593                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1594                     "resume token is corrupt (invalid format)"));
1595                 return (NULL);
1596         }
1597
1598         if (version != ZFS_SEND_RESUME_TOKEN_VERSION) {
1599                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1600                     "resume token is corrupt (invalid version %u)"),
1601                     version);
1602                 return (NULL);
1603         }
1604
1605         /* convert hexadecimal representation to binary */
1606         token = strrchr(token, '-') + 1;
1607         int len = strlen(token) / 2;
1608         unsigned char *compressed = zfs_alloc(hdl, len);
1609         for (i = 0; i < len; i++) {
1610                 nread = sscanf(token + i * 2, "%2hhx", compressed + i);
1611                 if (nread != 1) {
1612                         free(compressed);
1613                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1614                             "resume token is corrupt "
1615                             "(payload is not hex-encoded)"));
1616                         return (NULL);
1617                 }
1618         }
1619
1620         /* verify checksum */
1621         zio_cksum_t cksum;
1622         fletcher_4_native_varsize(compressed, len, &cksum);
1623         if (cksum.zc_word[0] != checksum) {
1624                 free(compressed);
1625                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1626                     "resume token is corrupt (incorrect checksum)"));
1627                 return (NULL);
1628         }
1629
1630         /* uncompress */
1631         void *packed = zfs_alloc(hdl, packed_len);
1632         uLongf packed_len_long = packed_len;
1633         if (uncompress(packed, &packed_len_long, compressed, len) != Z_OK ||
1634             packed_len_long != packed_len) {
1635                 free(packed);
1636                 free(compressed);
1637                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1638                     "resume token is corrupt (decompression failed)"));
1639                 return (NULL);
1640         }
1641
1642         /* unpack nvlist */
1643         nvlist_t *nv;
1644         int error = nvlist_unpack(packed, packed_len, &nv, KM_SLEEP);
1645         free(packed);
1646         free(compressed);
1647         if (error != 0) {
1648                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1649                     "resume token is corrupt (nvlist_unpack failed)"));
1650                 return (NULL);
1651         }
1652         return (nv);
1653 }
1654
1655 int
1656 zfs_send_resume(libzfs_handle_t *hdl, sendflags_t *flags, int outfd,
1657     const char *resume_token)
1658 {
1659         char errbuf[1024];
1660         char *toname;
1661         char *fromname = NULL;
1662         uint64_t resumeobj, resumeoff, toguid, fromguid, bytes;
1663         zfs_handle_t *zhp;
1664         int error = 0;
1665         char name[ZFS_MAX_DATASET_NAME_LEN];
1666         enum lzc_send_flags lzc_flags = 0;
1667         FILE *fout = (flags->verbose && flags->dryrun) ? stdout : stderr;
1668
1669         (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
1670             "cannot resume send"));
1671
1672         nvlist_t *resume_nvl =
1673             zfs_send_resume_token_to_nvlist(hdl, resume_token);
1674         if (resume_nvl == NULL) {
1675                 /*
1676                  * zfs_error_aux has already been set by
1677                  * zfs_send_resume_token_to_nvlist
1678                  */
1679                 return (zfs_error(hdl, EZFS_FAULT, errbuf));
1680         }
1681         if (flags->verbose) {
1682                 (void) fprintf(fout, dgettext(TEXT_DOMAIN,
1683                     "resume token contents:\n"));
1684                 nvlist_print(fout, resume_nvl);
1685         }
1686
1687         if (nvlist_lookup_string(resume_nvl, "toname", &toname) != 0 ||
1688             nvlist_lookup_uint64(resume_nvl, "object", &resumeobj) != 0 ||
1689             nvlist_lookup_uint64(resume_nvl, "offset", &resumeoff) != 0 ||
1690             nvlist_lookup_uint64(resume_nvl, "bytes", &bytes) != 0 ||
1691             nvlist_lookup_uint64(resume_nvl, "toguid", &toguid) != 0) {
1692                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1693                     "resume token is corrupt"));
1694                 return (zfs_error(hdl, EZFS_FAULT, errbuf));
1695         }
1696         fromguid = 0;
1697         (void) nvlist_lookup_uint64(resume_nvl, "fromguid", &fromguid);
1698
1699         if (flags->largeblock || nvlist_exists(resume_nvl, "largeblockok"))
1700                 lzc_flags |= LZC_SEND_FLAG_LARGE_BLOCK;
1701         if (flags->embed_data || nvlist_exists(resume_nvl, "embedok"))
1702                 lzc_flags |= LZC_SEND_FLAG_EMBED_DATA;
1703         if (flags->compress || nvlist_exists(resume_nvl, "compressok"))
1704                 lzc_flags |= LZC_SEND_FLAG_COMPRESS;
1705         if (flags->raw || nvlist_exists(resume_nvl, "rawok"))
1706                 lzc_flags |= LZC_SEND_FLAG_RAW;
1707
1708         if (guid_to_name(hdl, toname, toguid, B_FALSE, name) != 0) {
1709                 if (zfs_dataset_exists(hdl, toname, ZFS_TYPE_DATASET)) {
1710                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1711                             "'%s' is no longer the same snapshot used in "
1712                             "the initial send"), toname);
1713                 } else {
1714                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1715                             "'%s' used in the initial send no longer exists"),
1716                             toname);
1717                 }
1718                 return (zfs_error(hdl, EZFS_BADPATH, errbuf));
1719         }
1720         zhp = zfs_open(hdl, name, ZFS_TYPE_DATASET);
1721         if (zhp == NULL) {
1722                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1723                     "unable to access '%s'"), name);
1724                 return (zfs_error(hdl, EZFS_BADPATH, errbuf));
1725         }
1726
1727         if (fromguid != 0) {
1728                 if (guid_to_name(hdl, toname, fromguid, B_TRUE, name) != 0) {
1729                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1730                             "incremental source %#llx no longer exists"),
1731                             (longlong_t)fromguid);
1732                         return (zfs_error(hdl, EZFS_BADPATH, errbuf));
1733                 }
1734                 fromname = name;
1735         }
1736
1737         if (flags->verbose) {
1738                 uint64_t size = 0;
1739                 error = lzc_send_space(zhp->zfs_name, fromname,
1740                     lzc_flags, &size);
1741                 if (error == 0)
1742                         size = MAX(0, (int64_t)(size - bytes));
1743                 send_print_verbose(fout, zhp->zfs_name, fromname,
1744                     size, flags->parsable);
1745         }
1746
1747         if (!flags->dryrun) {
1748                 progress_arg_t pa = { 0 };
1749                 pthread_t tid;
1750                 /*
1751                  * If progress reporting is requested, spawn a new thread to
1752                  * poll ZFS_IOC_SEND_PROGRESS at a regular interval.
1753                  */
1754                 if (flags->progress) {
1755                         pa.pa_zhp = zhp;
1756                         pa.pa_fd = outfd;
1757                         pa.pa_parsable = flags->parsable;
1758
1759                         error = pthread_create(&tid, NULL,
1760                             send_progress_thread, &pa);
1761                         if (error != 0) {
1762                                 zfs_close(zhp);
1763                                 return (error);
1764                         }
1765                 }
1766
1767                 error = lzc_send_resume(zhp->zfs_name, fromname, outfd,
1768                     lzc_flags, resumeobj, resumeoff);
1769
1770                 if (flags->progress) {
1771                         (void) pthread_cancel(tid);
1772                         (void) pthread_join(tid, NULL);
1773                 }
1774
1775                 char errbuf[1024];
1776                 (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
1777                     "warning: cannot send '%s'"), zhp->zfs_name);
1778
1779                 zfs_close(zhp);
1780
1781                 switch (error) {
1782                 case 0:
1783                         return (0);
1784                 case EACCES:
1785                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
1786                             "source key must be loaded"));
1787                         return (zfs_error(hdl, EZFS_CRYPTOFAILED, errbuf));
1788
1789                 case EXDEV:
1790                 case ENOENT:
1791                 case EDQUOT:
1792                 case EFBIG:
1793                 case EIO:
1794                 case ENOLINK:
1795                 case ENOSPC:
1796                 case ENOSTR:
1797                 case ENXIO:
1798                 case EPIPE:
1799                 case ERANGE:
1800                 case EFAULT:
1801                 case EROFS:
1802                         zfs_error_aux(hdl, strerror(errno));
1803                         return (zfs_error(hdl, EZFS_BADBACKUP, errbuf));
1804
1805                 default:
1806                         return (zfs_standard_error(hdl, errno, errbuf));
1807                 }
1808         }
1809
1810
1811         zfs_close(zhp);
1812
1813         return (error);
1814 }
1815
1816 /*
1817  * Generate a send stream for the dataset identified by the argument zhp.
1818  *
1819  * The content of the send stream is the snapshot identified by
1820  * 'tosnap'.  Incremental streams are requested in two ways:
1821  *     - from the snapshot identified by "fromsnap" (if non-null) or
1822  *     - from the origin of the dataset identified by zhp, which must
1823  *       be a clone.  In this case, "fromsnap" is null and "fromorigin"
1824  *       is TRUE.
1825  *
1826  * The send stream is recursive (i.e. dumps a hierarchy of snapshots) and
1827  * uses a special header (with a hdrtype field of DMU_COMPOUNDSTREAM)
1828  * if "replicate" is set.  If "doall" is set, dump all the intermediate
1829  * snapshots. The DMU_COMPOUNDSTREAM header is used in the "doall"
1830  * case too. If "props" is set, send properties.
1831  */
1832 int
1833 zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
1834     sendflags_t *flags, int outfd, snapfilter_cb_t filter_func,
1835     void *cb_arg, nvlist_t **debugnvp)
1836 {
1837         char errbuf[1024];
1838         send_dump_data_t sdd = { 0 };
1839         int err = 0;
1840         nvlist_t *fss = NULL;
1841         avl_tree_t *fsavl = NULL;
1842         static uint64_t holdseq;
1843         int spa_version;
1844         pthread_t tid = 0;
1845         int pipefd[2];
1846         dedup_arg_t dda = { 0 };
1847         int featureflags = 0;
1848         FILE *fout;
1849
1850         (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
1851             "cannot send '%s'"), zhp->zfs_name);
1852
1853         if (fromsnap && fromsnap[0] == '\0') {
1854                 zfs_error_aux(zhp->zfs_hdl, dgettext(TEXT_DOMAIN,
1855                     "zero-length incremental source"));
1856                 return (zfs_error(zhp->zfs_hdl, EZFS_NOENT, errbuf));
1857         }
1858
1859         if (zhp->zfs_type == ZFS_TYPE_FILESYSTEM) {
1860                 uint64_t version;
1861                 version = zfs_prop_get_int(zhp, ZFS_PROP_VERSION);
1862                 if (version >= ZPL_VERSION_SA) {
1863                         featureflags |= DMU_BACKUP_FEATURE_SA_SPILL;
1864                 }
1865         }
1866
1867         /*
1868          * Start the dedup thread if this is a dedup stream. We do not bother
1869          * doing this if this a raw send of an encrypted dataset with dedup off
1870          * because normal encrypted blocks won't dedup.
1871          */
1872         if (flags->dedup && !flags->dryrun && !(flags->raw &&
1873             zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION) != ZIO_CRYPT_OFF &&
1874             zfs_prop_get_int(zhp, ZFS_PROP_DEDUP) == ZIO_CHECKSUM_OFF)) {
1875                 featureflags |= (DMU_BACKUP_FEATURE_DEDUP |
1876                     DMU_BACKUP_FEATURE_DEDUPPROPS);
1877                 if ((err = socketpair(AF_UNIX, SOCK_STREAM, 0, pipefd)) != 0) {
1878                         zfs_error_aux(zhp->zfs_hdl, strerror(errno));
1879                         return (zfs_error(zhp->zfs_hdl, EZFS_PIPEFAILED,
1880                             errbuf));
1881                 }
1882                 dda.outputfd = outfd;
1883                 dda.inputfd = pipefd[1];
1884                 dda.dedup_hdl = zhp->zfs_hdl;
1885                 if ((err = pthread_create(&tid, NULL, cksummer, &dda)) != 0) {
1886                         (void) close(pipefd[0]);
1887                         (void) close(pipefd[1]);
1888                         zfs_error_aux(zhp->zfs_hdl, strerror(errno));
1889                         return (zfs_error(zhp->zfs_hdl,
1890                             EZFS_THREADCREATEFAILED, errbuf));
1891                 }
1892         }
1893
1894         if (flags->replicate || flags->doall || flags->props || flags->backup) {
1895                 dmu_replay_record_t drr = { 0 };
1896                 char *packbuf = NULL;
1897                 size_t buflen = 0;
1898                 zio_cksum_t zc;
1899
1900                 ZIO_SET_CHECKSUM(&zc, 0, 0, 0, 0);
1901
1902                 if (flags->replicate || flags->props || flags->backup) {
1903                         nvlist_t *hdrnv;
1904
1905                         VERIFY(0 == nvlist_alloc(&hdrnv, NV_UNIQUE_NAME, 0));
1906                         if (fromsnap) {
1907                                 VERIFY(0 == nvlist_add_string(hdrnv,
1908                                     "fromsnap", fromsnap));
1909                         }
1910                         VERIFY(0 == nvlist_add_string(hdrnv, "tosnap", tosnap));
1911                         if (!flags->replicate) {
1912                                 VERIFY(0 == nvlist_add_boolean(hdrnv,
1913                                     "not_recursive"));
1914                         }
1915                         if (flags->raw) {
1916                                 VERIFY(0 == nvlist_add_boolean(hdrnv, "raw"));
1917                         }
1918
1919                         err = gather_nvlist(zhp->zfs_hdl, zhp->zfs_name,
1920                             fromsnap, tosnap, flags->replicate, flags->raw,
1921                             flags->verbose, flags->backup, &fss, &fsavl);
1922                         if (err)
1923                                 goto err_out;
1924                         VERIFY(0 == nvlist_add_nvlist(hdrnv, "fss", fss));
1925                         err = nvlist_pack(hdrnv, &packbuf, &buflen,
1926                             NV_ENCODE_XDR, 0);
1927                         if (debugnvp)
1928                                 *debugnvp = hdrnv;
1929                         else
1930                                 nvlist_free(hdrnv);
1931                         if (err)
1932                                 goto stderr_out;
1933                 }
1934
1935                 if (!flags->dryrun) {
1936                         /* write first begin record */
1937                         drr.drr_type = DRR_BEGIN;
1938                         drr.drr_u.drr_begin.drr_magic = DMU_BACKUP_MAGIC;
1939                         DMU_SET_STREAM_HDRTYPE(drr.drr_u.drr_begin.
1940                             drr_versioninfo, DMU_COMPOUNDSTREAM);
1941                         DMU_SET_FEATUREFLAGS(drr.drr_u.drr_begin.
1942                             drr_versioninfo, featureflags);
1943                         if (snprintf(drr.drr_u.drr_begin.drr_toname,
1944                             sizeof (drr.drr_u.drr_begin.drr_toname),
1945                             "%s@%s", zhp->zfs_name, tosnap) >=
1946                             sizeof (drr.drr_u.drr_begin.drr_toname)) {
1947                                 err = EINVAL;
1948                                 goto stderr_out;
1949                         }
1950                         drr.drr_payloadlen = buflen;
1951
1952                         err = dump_record(&drr, packbuf, buflen, &zc, outfd);
1953                         free(packbuf);
1954                         if (err != 0)
1955                                 goto stderr_out;
1956
1957                         /* write end record */
1958                         bzero(&drr, sizeof (drr));
1959                         drr.drr_type = DRR_END;
1960                         drr.drr_u.drr_end.drr_checksum = zc;
1961                         err = write(outfd, &drr, sizeof (drr));
1962                         if (err == -1) {
1963                                 err = errno;
1964                                 goto stderr_out;
1965                         }
1966
1967                         err = 0;
1968                 }
1969         }
1970
1971         /* dump each stream */
1972         sdd.fromsnap = fromsnap;
1973         sdd.tosnap = tosnap;
1974         if (tid != 0)
1975                 sdd.outfd = pipefd[0];
1976         else
1977                 sdd.outfd = outfd;
1978         sdd.replicate = flags->replicate;
1979         sdd.doall = flags->doall;
1980         sdd.fromorigin = flags->fromorigin;
1981         sdd.fss = fss;
1982         sdd.fsavl = fsavl;
1983         sdd.verbose = flags->verbose;
1984         sdd.parsable = flags->parsable;
1985         sdd.progress = flags->progress;
1986         sdd.dryrun = flags->dryrun;
1987         sdd.large_block = flags->largeblock;
1988         sdd.embed_data = flags->embed_data;
1989         sdd.compress = flags->compress;
1990         sdd.raw = flags->raw;
1991         sdd.filter_cb = filter_func;
1992         sdd.filter_cb_arg = cb_arg;
1993         if (debugnvp)
1994                 sdd.debugnv = *debugnvp;
1995         if (sdd.verbose && sdd.dryrun)
1996                 sdd.std_out = B_TRUE;
1997         fout = sdd.std_out ? stdout : stderr;
1998
1999         /*
2000          * Some flags require that we place user holds on the datasets that are
2001          * being sent so they don't get destroyed during the send. We can skip
2002          * this step if the pool is imported read-only since the datasets cannot
2003          * be destroyed.
2004          */
2005         if (!flags->dryrun && !zpool_get_prop_int(zfs_get_pool_handle(zhp),
2006             ZPOOL_PROP_READONLY, NULL) &&
2007             zfs_spa_version(zhp, &spa_version) == 0 &&
2008             spa_version >= SPA_VERSION_USERREFS &&
2009             (flags->doall || flags->replicate)) {
2010                 ++holdseq;
2011                 (void) snprintf(sdd.holdtag, sizeof (sdd.holdtag),
2012                     ".send-%d-%llu", getpid(), (u_longlong_t)holdseq);
2013                 sdd.cleanup_fd = open(ZFS_DEV, O_RDWR);
2014                 if (sdd.cleanup_fd < 0) {
2015                         err = errno;
2016                         goto stderr_out;
2017                 }
2018                 sdd.snapholds = fnvlist_alloc();
2019         } else {
2020                 sdd.cleanup_fd = -1;
2021                 sdd.snapholds = NULL;
2022         }
2023         if (flags->verbose || sdd.snapholds != NULL) {
2024                 /*
2025                  * Do a verbose no-op dry run to get all the verbose output
2026                  * or to gather snapshot hold's before generating any data,
2027                  * then do a non-verbose real run to generate the streams.
2028                  */
2029                 sdd.dryrun = B_TRUE;
2030                 err = dump_filesystems(zhp, &sdd);
2031
2032                 if (err != 0)
2033                         goto stderr_out;
2034
2035                 if (flags->verbose) {
2036                         if (flags->parsable) {
2037                                 (void) fprintf(fout, "size\t%llu\n",
2038                                     (longlong_t)sdd.size);
2039                         } else {
2040                                 char buf[16];
2041                                 zfs_nicebytes(sdd.size, buf, sizeof (buf));
2042                                 (void) fprintf(fout, dgettext(TEXT_DOMAIN,
2043                                     "total estimated size is %s\n"), buf);
2044                         }
2045                 }
2046
2047                 /* Ensure no snaps found is treated as an error. */
2048                 if (!sdd.seento) {
2049                         err = ENOENT;
2050                         goto err_out;
2051                 }
2052
2053                 /* Skip the second run if dryrun was requested. */
2054                 if (flags->dryrun)
2055                         goto err_out;
2056
2057                 if (sdd.snapholds != NULL) {
2058                         err = zfs_hold_nvl(zhp, sdd.cleanup_fd, sdd.snapholds);
2059                         if (err != 0)
2060                                 goto stderr_out;
2061
2062                         fnvlist_free(sdd.snapholds);
2063                         sdd.snapholds = NULL;
2064                 }
2065
2066                 sdd.dryrun = B_FALSE;
2067                 sdd.verbose = B_FALSE;
2068         }
2069
2070         err = dump_filesystems(zhp, &sdd);
2071         fsavl_destroy(fsavl);
2072         nvlist_free(fss);
2073
2074         /* Ensure no snaps found is treated as an error. */
2075         if (err == 0 && !sdd.seento)
2076                 err = ENOENT;
2077
2078         if (tid != 0) {
2079                 if (err != 0)
2080                         (void) pthread_cancel(tid);
2081                 (void) close(pipefd[0]);
2082                 (void) pthread_join(tid, NULL);
2083         }
2084
2085         if (sdd.cleanup_fd != -1) {
2086                 VERIFY(0 == close(sdd.cleanup_fd));
2087                 sdd.cleanup_fd = -1;
2088         }
2089
2090         if (!flags->dryrun && (flags->replicate || flags->doall ||
2091             flags->props || flags->backup)) {
2092                 /*
2093                  * write final end record.  NB: want to do this even if
2094                  * there was some error, because it might not be totally
2095                  * failed.
2096                  */
2097                 dmu_replay_record_t drr = { 0 };
2098                 drr.drr_type = DRR_END;
2099                 if (write(outfd, &drr, sizeof (drr)) == -1) {
2100                         return (zfs_standard_error(zhp->zfs_hdl,
2101                             errno, errbuf));
2102                 }
2103         }
2104
2105         return (err || sdd.err);
2106
2107 stderr_out:
2108         err = zfs_standard_error(zhp->zfs_hdl, err, errbuf);
2109 err_out:
2110         fsavl_destroy(fsavl);
2111         nvlist_free(fss);
2112         fnvlist_free(sdd.snapholds);
2113
2114         if (sdd.cleanup_fd != -1)
2115                 VERIFY(0 == close(sdd.cleanup_fd));
2116         if (tid != 0) {
2117                 (void) pthread_cancel(tid);
2118                 (void) close(pipefd[0]);
2119                 (void) pthread_join(tid, NULL);
2120         }
2121         return (err);
2122 }
2123
2124 int
2125 zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t flags)
2126 {
2127         int err = 0;
2128         libzfs_handle_t *hdl = zhp->zfs_hdl;
2129         enum lzc_send_flags lzc_flags = 0;
2130         FILE *fout = (flags.verbose && flags.dryrun) ? stdout : stderr;
2131         char errbuf[1024];
2132
2133         if (flags.largeblock)
2134                 lzc_flags |= LZC_SEND_FLAG_LARGE_BLOCK;
2135         if (flags.embed_data)
2136                 lzc_flags |= LZC_SEND_FLAG_EMBED_DATA;
2137         if (flags.compress)
2138                 lzc_flags |= LZC_SEND_FLAG_COMPRESS;
2139         if (flags.raw)
2140                 lzc_flags |= LZC_SEND_FLAG_RAW;
2141
2142         if (flags.verbose) {
2143                 uint64_t size = 0;
2144                 err = lzc_send_space(zhp->zfs_name, from, lzc_flags, &size);
2145                 if (err == 0) {
2146                         send_print_verbose(fout, zhp->zfs_name, from, size,
2147                             flags.parsable);
2148                 } else {
2149                         (void) fprintf(stderr, "Cannot estimate send size: "
2150                             "%s\n", strerror(errno));
2151                 }
2152         }
2153
2154         if (flags.dryrun)
2155                 return (err);
2156
2157         (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
2158             "warning: cannot send '%s'"), zhp->zfs_name);
2159
2160         err = lzc_send(zhp->zfs_name, from, fd, lzc_flags);
2161         if (err != 0) {
2162                 switch (errno) {
2163                 case EXDEV:
2164                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2165                             "not an earlier snapshot from the same fs"));
2166                         return (zfs_error(hdl, EZFS_CROSSTARGET, errbuf));
2167
2168                 case ENOENT:
2169                 case ESRCH:
2170                         if (lzc_exists(zhp->zfs_name)) {
2171                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2172                                     "incremental source (%s) does not exist"),
2173                                     from);
2174                         }
2175                         return (zfs_error(hdl, EZFS_NOENT, errbuf));
2176
2177                 case EACCES:
2178                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2179                             "dataset key must be loaded"));
2180                         return (zfs_error(hdl, EZFS_CRYPTOFAILED, errbuf));
2181
2182                 case EBUSY:
2183                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2184                             "target is busy; if a filesystem, "
2185                             "it must not be mounted"));
2186                         return (zfs_error(hdl, EZFS_BUSY, errbuf));
2187
2188                 case EDQUOT:
2189                 case EFBIG:
2190                 case EIO:
2191                 case ENOLINK:
2192                 case ENOSPC:
2193                 case ENOSTR:
2194                 case ENXIO:
2195                 case EPIPE:
2196                 case ERANGE:
2197                 case EFAULT:
2198                 case EROFS:
2199                         zfs_error_aux(hdl, strerror(errno));
2200                         return (zfs_error(hdl, EZFS_BADBACKUP, errbuf));
2201
2202                 default:
2203                         return (zfs_standard_error(hdl, errno, errbuf));
2204                 }
2205         }
2206         return (err != 0);
2207 }
2208
2209 /*
2210  * Routines specific to "zfs recv"
2211  */
2212
2213 static int
2214 recv_read(libzfs_handle_t *hdl, int fd, void *buf, int ilen,
2215     boolean_t byteswap, zio_cksum_t *zc)
2216 {
2217         char *cp = buf;
2218         int rv;
2219         int len = ilen;
2220
2221         assert(ilen <= SPA_MAXBLOCKSIZE);
2222
2223         do {
2224                 rv = read(fd, cp, len);
2225                 cp += rv;
2226                 len -= rv;
2227         } while (rv > 0);
2228
2229         if (rv < 0 || len != 0) {
2230                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
2231                     "failed to read from stream"));
2232                 return (zfs_error(hdl, EZFS_BADSTREAM, dgettext(TEXT_DOMAIN,
2233                     "cannot receive")));
2234         }
2235
2236         if (zc) {
2237                 if (byteswap)
2238                         fletcher_4_incremental_byteswap(buf, ilen, zc);
2239                 else
2240                         fletcher_4_incremental_native(buf, ilen, zc);
2241         }
2242         return (0);
2243 }
2244
2245 static int
2246 recv_read_nvlist(libzfs_handle_t *hdl, int fd, int len, nvlist_t **nvp,
2247     boolean_t byteswap, zio_cksum_t *zc)
2248 {
2249         char *buf;
2250         int err;
2251
2252         buf = zfs_alloc(hdl, len);
2253         if (buf == NULL)
2254                 return (ENOMEM);
2255
2256         err = recv_read(hdl, fd, buf, len, byteswap, zc);
2257         if (err != 0) {
2258                 free(buf);
2259                 return (err);
2260         }
2261
2262         err = nvlist_unpack(buf, len, nvp, 0);
2263         free(buf);
2264         if (err != 0) {
2265                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
2266                     "stream (malformed nvlist)"));
2267                 return (EINVAL);
2268         }
2269         return (0);
2270 }
2271
2272 /*
2273  * Returns the grand origin (origin of origin of origin...) of a given handle.
2274  * If this dataset is not a clone, it simply returns a copy of the original
2275  * handle.
2276  */
2277 static zfs_handle_t *
2278 recv_open_grand_origin(zfs_handle_t *zhp)
2279 {
2280         char origin[ZFS_MAX_DATASET_NAME_LEN];
2281         zprop_source_t src;
2282         zfs_handle_t *ozhp = zfs_handle_dup(zhp);
2283
2284         while (ozhp != NULL) {
2285                 if (zfs_prop_get(ozhp, ZFS_PROP_ORIGIN, origin,
2286                     sizeof (origin), &src, NULL, 0, B_FALSE) != 0)
2287                         break;
2288
2289                 (void) zfs_close(ozhp);
2290                 ozhp = zfs_open(zhp->zfs_hdl, origin, ZFS_TYPE_FILESYSTEM);
2291         }
2292
2293         return (ozhp);
2294 }
2295
2296 static int
2297 recv_rename_impl(zfs_handle_t *zhp, const char *name, const char *newname)
2298 {
2299         int err;
2300         zfs_handle_t *ozhp = NULL;
2301
2302         /*
2303          * Attempt to rename the dataset. If it fails with EACCES we have
2304          * attempted to rename the dataset outside of its encryption root.
2305          * Force the dataset to become an encryption root and try again.
2306          */
2307         err = lzc_rename(name, newname);
2308         if (err == EACCES) {
2309                 ozhp = recv_open_grand_origin(zhp);
2310                 if (ozhp == NULL) {
2311                         err = ENOENT;
2312                         goto out;
2313                 }
2314
2315                 err = lzc_change_key(ozhp->zfs_name, DCP_CMD_FORCE_NEW_KEY,
2316                     NULL, NULL, 0);
2317                 if (err != 0)
2318                         goto out;
2319
2320                 err = lzc_rename(name, newname);
2321         }
2322
2323 out:
2324         if (ozhp != NULL)
2325                 zfs_close(ozhp);
2326         return (err);
2327 }
2328
2329 static int
2330 recv_rename(libzfs_handle_t *hdl, const char *name, const char *tryname,
2331     int baselen, char *newname, recvflags_t *flags)
2332 {
2333         static int seq;
2334         int err;
2335         prop_changelist_t *clp = NULL;
2336         zfs_handle_t *zhp = NULL;
2337
2338         zhp = zfs_open(hdl, name, ZFS_TYPE_DATASET);
2339         if (zhp == NULL) {
2340                 err = -1;
2341                 goto out;
2342         }
2343         clp = changelist_gather(zhp, ZFS_PROP_NAME, 0,
2344             flags->force ? MS_FORCE : 0);
2345         if (clp == NULL) {
2346                 err = -1;
2347                 goto out;
2348         }
2349         err = changelist_prefix(clp);
2350         if (err)
2351                 goto out;
2352
2353         if (tryname) {
2354                 (void) strcpy(newname, tryname);
2355                 if (flags->verbose) {
2356                         (void) printf("attempting rename %s to %s\n",
2357                             name, newname);
2358                 }
2359                 err = recv_rename_impl(zhp, name, newname);
2360                 if (err == 0)
2361                         changelist_rename(clp, name, tryname);
2362         } else {
2363                 err = ENOENT;
2364         }
2365
2366         if (err != 0 && strncmp(name + baselen, "recv-", 5) != 0) {
2367                 seq++;
2368
2369                 (void) snprintf(newname, ZFS_MAX_DATASET_NAME_LEN,
2370                     "%.*srecv-%u-%u", baselen, name, getpid(), seq);
2371
2372                 if (flags->verbose) {
2373                         (void) printf("failed - trying rename %s to %s\n",
2374                             name, newname);
2375                 }
2376                 err = recv_rename_impl(zhp, name, newname);
2377                 if (err == 0)
2378                         changelist_rename(clp, name, newname);
2379                 if (err && flags->verbose) {
2380                         (void) printf("failed (%u) - "
2381                             "will try again on next pass\n", errno);
2382                 }
2383                 err = EAGAIN;
2384         } else if (flags->verbose) {
2385                 if (err == 0)
2386                         (void) printf("success\n");
2387                 else
2388                         (void) printf("failed (%u)\n", errno);
2389         }
2390
2391         (void) changelist_postfix(clp);
2392
2393 out:
2394         if (clp != NULL)
2395                 changelist_free(clp);
2396         if (zhp != NULL)
2397                 zfs_close(zhp);
2398
2399         return (err);
2400 }
2401
2402 static int
2403 recv_promote(libzfs_handle_t *hdl, const char *fsname,
2404     const char *origin_fsname, recvflags_t *flags)
2405 {
2406         int err;
2407         zfs_cmd_t zc = {"\0"};
2408         zfs_handle_t *zhp = NULL, *ozhp = NULL;
2409
2410         if (flags->verbose)
2411                 (void) printf("promoting %s\n", fsname);
2412
2413         (void) strlcpy(zc.zc_value, origin_fsname, sizeof (zc.zc_value));
2414         (void) strlcpy(zc.zc_name, fsname, sizeof (zc.zc_name));
2415
2416         /*
2417          * Attempt to promote the dataset. If it fails with EACCES the
2418          * promotion would cause this dataset to leave its encryption root.
2419          * Force the origin to become an encryption root and try again.
2420          */
2421         err = zfs_ioctl(hdl, ZFS_IOC_PROMOTE, &zc);
2422         if (err == EACCES) {
2423                 zhp = zfs_open(hdl, fsname, ZFS_TYPE_DATASET);
2424                 if (zhp == NULL) {
2425                         err = -1;
2426                         goto out;
2427                 }
2428
2429                 ozhp = recv_open_grand_origin(zhp);
2430                 if (ozhp == NULL) {
2431                         err = -1;
2432                         goto out;
2433                 }
2434
2435                 err = lzc_change_key(ozhp->zfs_name, DCP_CMD_FORCE_NEW_KEY,
2436                     NULL, NULL, 0);
2437                 if (err != 0)
2438                         goto out;
2439
2440                 err = zfs_ioctl(hdl, ZFS_IOC_PROMOTE, &zc);
2441         }
2442
2443 out:
2444         if (zhp != NULL)
2445                 zfs_close(zhp);
2446         if (ozhp != NULL)
2447                 zfs_close(ozhp);
2448
2449         return (err);
2450 }
2451
2452 static int
2453 recv_destroy(libzfs_handle_t *hdl, const char *name, int baselen,
2454     char *newname, recvflags_t *flags)
2455 {
2456         int err = 0;
2457         prop_changelist_t *clp;
2458         zfs_handle_t *zhp;
2459         boolean_t defer = B_FALSE;
2460         int spa_version;
2461
2462         zhp = zfs_open(hdl, name, ZFS_TYPE_DATASET);
2463         if (zhp == NULL)
2464                 return (-1);
2465         clp = changelist_gather(zhp, ZFS_PROP_NAME, 0,
2466             flags->force ? MS_FORCE : 0);
2467         if (zfs_get_type(zhp) == ZFS_TYPE_SNAPSHOT &&
2468             zfs_spa_version(zhp, &spa_version) == 0 &&
2469             spa_version >= SPA_VERSION_USERREFS)
2470                 defer = B_TRUE;
2471         zfs_close(zhp);
2472         if (clp == NULL)
2473                 return (-1);
2474         err = changelist_prefix(clp);
2475         if (err)
2476                 return (err);
2477
2478         if (flags->verbose)
2479                 (void) printf("attempting destroy %s\n", name);
2480         if (zhp->zfs_type == ZFS_TYPE_SNAPSHOT) {
2481                 nvlist_t *nv = fnvlist_alloc();
2482                 fnvlist_add_boolean(nv, name);
2483                 err = lzc_destroy_snaps(nv, defer, NULL);
2484                 fnvlist_free(nv);
2485         } else {
2486                 err = lzc_destroy(name);
2487         }
2488         if (err == 0) {
2489                 if (flags->verbose)
2490                         (void) printf("success\n");
2491                 changelist_remove(clp, name);
2492         }
2493
2494         (void) changelist_postfix(clp);
2495         changelist_free(clp);
2496
2497         /*
2498          * Deferred destroy might destroy the snapshot or only mark it to be
2499          * destroyed later, and it returns success in either case.
2500          */
2501         if (err != 0 || (defer && zfs_dataset_exists(hdl, name,
2502             ZFS_TYPE_SNAPSHOT))) {
2503                 err = recv_rename(hdl, name, NULL, baselen, newname, flags);
2504         }
2505
2506         return (err);
2507 }
2508
2509 typedef struct guid_to_name_data {
2510         uint64_t guid;
2511         boolean_t bookmark_ok;
2512         char *name;
2513         char *skip;
2514 } guid_to_name_data_t;
2515
2516 static int
2517 guid_to_name_cb(zfs_handle_t *zhp, void *arg)
2518 {
2519         guid_to_name_data_t *gtnd = arg;
2520         const char *slash;
2521         int err;
2522
2523         if (gtnd->skip != NULL &&
2524             (slash = strrchr(zhp->zfs_name, '/')) != NULL &&
2525             strcmp(slash + 1, gtnd->skip) == 0) {
2526                 zfs_close(zhp);
2527                 return (0);
2528         }
2529
2530         if (zfs_prop_get_int(zhp, ZFS_PROP_GUID) == gtnd->guid) {
2531                 (void) strcpy(gtnd->name, zhp->zfs_name);
2532                 zfs_close(zhp);
2533                 return (EEXIST);
2534         }
2535
2536         err = zfs_iter_children(zhp, guid_to_name_cb, gtnd);
2537         if (err != EEXIST && gtnd->bookmark_ok)
2538                 err = zfs_iter_bookmarks(zhp, guid_to_name_cb, gtnd);
2539         zfs_close(zhp);
2540         return (err);
2541 }
2542
2543 /*
2544  * Attempt to find the local dataset associated with this guid.  In the case of
2545  * multiple matches, we attempt to find the "best" match by searching
2546  * progressively larger portions of the hierarchy.  This allows one to send a
2547  * tree of datasets individually and guarantee that we will find the source
2548  * guid within that hierarchy, even if there are multiple matches elsewhere.
2549  */
2550 static int
2551 guid_to_name(libzfs_handle_t *hdl, const char *parent, uint64_t guid,
2552     boolean_t bookmark_ok, char *name)
2553 {
2554         char pname[ZFS_MAX_DATASET_NAME_LEN];
2555         guid_to_name_data_t gtnd;
2556
2557         gtnd.guid = guid;
2558         gtnd.bookmark_ok = bookmark_ok;
2559         gtnd.name = name;
2560         gtnd.skip = NULL;
2561
2562         /*
2563          * Search progressively larger portions of the hierarchy, starting
2564          * with the filesystem specified by 'parent'.  This will
2565          * select the "most local" version of the origin snapshot in the case
2566          * that there are multiple matching snapshots in the system.
2567          */
2568         (void) strlcpy(pname, parent, sizeof (pname));
2569         char *cp = strrchr(pname, '@');
2570         if (cp == NULL)
2571                 cp = strchr(pname, '\0');
2572         for (; cp != NULL; cp = strrchr(pname, '/')) {
2573                 /* Chop off the last component and open the parent */
2574                 *cp = '\0';
2575                 zfs_handle_t *zhp = make_dataset_handle(hdl, pname);
2576
2577                 if (zhp == NULL)
2578                         continue;
2579                 int err = guid_to_name_cb(zfs_handle_dup(zhp), &gtnd);
2580                 if (err != EEXIST)
2581                         err = zfs_iter_children(zhp, guid_to_name_cb, &gtnd);
2582                 if (err != EEXIST && bookmark_ok)
2583                         err = zfs_iter_bookmarks(zhp, guid_to_name_cb, &gtnd);
2584                 zfs_close(zhp);
2585                 if (err == EEXIST)
2586                         return (0);
2587
2588                 /*
2589                  * Remember the last portion of the dataset so we skip it next
2590                  * time through (as we've already searched that portion of the
2591                  * hierarchy).
2592                  */
2593                 gtnd.skip = strrchr(pname, '/') + 1;
2594         }
2595
2596         return (ENOENT);
2597 }
2598
2599 /*
2600  * Return +1 if guid1 is before guid2, 0 if they are the same, and -1 if
2601  * guid1 is after guid2.
2602  */
2603 static int
2604 created_before(libzfs_handle_t *hdl, avl_tree_t *avl,
2605     uint64_t guid1, uint64_t guid2)
2606 {
2607         nvlist_t *nvfs;
2608         char *fsname = NULL, *snapname = NULL;
2609         char buf[ZFS_MAX_DATASET_NAME_LEN];
2610         int rv;
2611         zfs_handle_t *guid1hdl, *guid2hdl;
2612         uint64_t create1, create2;
2613
2614         if (guid2 == 0)
2615                 return (0);
2616         if (guid1 == 0)
2617                 return (1);
2618
2619         nvfs = fsavl_find(avl, guid1, &snapname);
2620         VERIFY(0 == nvlist_lookup_string(nvfs, "name", &fsname));
2621         (void) snprintf(buf, sizeof (buf), "%s@%s", fsname, snapname);
2622         guid1hdl = zfs_open(hdl, buf, ZFS_TYPE_SNAPSHOT);
2623         if (guid1hdl == NULL)
2624                 return (-1);
2625
2626         nvfs = fsavl_find(avl, guid2, &snapname);
2627         VERIFY(0 == nvlist_lookup_string(nvfs, "name", &fsname));
2628         (void) snprintf(buf, sizeof (buf), "%s@%s", fsname, snapname);
2629         guid2hdl = zfs_open(hdl, buf, ZFS_TYPE_SNAPSHOT);
2630         if (guid2hdl == NULL) {
2631                 zfs_close(guid1hdl);
2632                 return (-1);
2633         }
2634
2635         create1 = zfs_prop_get_int(guid1hdl, ZFS_PROP_CREATETXG);
2636         create2 = zfs_prop_get_int(guid2hdl, ZFS_PROP_CREATETXG);
2637
2638         if (create1 < create2)
2639                 rv = -1;
2640         else if (create1 > create2)
2641                 rv = +1;
2642         else
2643                 rv = 0;
2644
2645         zfs_close(guid1hdl);
2646         zfs_close(guid2hdl);
2647
2648         return (rv);
2649 }
2650
2651 /*
2652  * This function reestablishes the heirarchy of encryption roots after a
2653  * recursive incremental receive has completed. This must be done after the
2654  * second call to recv_incremental_replication() has renamed and promoted all
2655  * sent datasets to their final locations in the dataset heriarchy.
2656  */
2657 static int
2658 recv_fix_encryption_heirarchy(libzfs_handle_t *hdl, const char *destname,
2659     nvlist_t *stream_nv, avl_tree_t *stream_avl)
2660 {
2661         int err;
2662         nvpair_t *fselem = NULL;
2663         nvlist_t *stream_fss;
2664         char *cp;
2665         char top_zfs[ZFS_MAX_DATASET_NAME_LEN];
2666
2667         (void) strcpy(top_zfs, destname);
2668         cp = strrchr(top_zfs, '@');
2669         if (cp != NULL)
2670                 *cp = '\0';
2671
2672         VERIFY(0 == nvlist_lookup_nvlist(stream_nv, "fss", &stream_fss));
2673
2674         while ((fselem = nvlist_next_nvpair(stream_fss, fselem)) != NULL) {
2675                 zfs_handle_t *zhp = NULL;
2676                 uint64_t crypt;
2677                 nvlist_t *snaps, *props, *stream_nvfs = NULL;
2678                 nvpair_t *snapel = NULL;
2679                 boolean_t is_encroot, is_clone, stream_encroot;
2680                 char *cp;
2681                 char *stream_keylocation = NULL;
2682                 char keylocation[MAXNAMELEN];
2683                 char fsname[ZFS_MAX_DATASET_NAME_LEN];
2684
2685                 keylocation[0] = '\0';
2686                 VERIFY(0 == nvpair_value_nvlist(fselem, &stream_nvfs));
2687                 VERIFY(0 == nvlist_lookup_nvlist(stream_nvfs, "snaps", &snaps));
2688                 VERIFY(0 == nvlist_lookup_nvlist(stream_nvfs, "props", &props));
2689                 stream_encroot = nvlist_exists(stream_nvfs, "is_encroot");
2690
2691                 /* find a snapshot from the stream that exists locally */
2692                 err = ENOENT;
2693                 while ((snapel = nvlist_next_nvpair(snaps, snapel)) != NULL) {
2694                         uint64_t guid;
2695
2696                         VERIFY(0 == nvpair_value_uint64(snapel, &guid));
2697                         err = guid_to_name(hdl, destname, guid, B_FALSE,
2698                             fsname);
2699                         if (err == 0)
2700                                 break;
2701                 }
2702
2703                 if (err != 0)
2704                         continue;
2705
2706                 cp = strchr(fsname, '@');
2707                 if (cp != NULL)
2708                         *cp = '\0';
2709
2710                 zhp = zfs_open(hdl, fsname, ZFS_TYPE_DATASET);
2711                 if (zhp == NULL) {
2712                         err = ENOENT;
2713                         goto error;
2714                 }
2715
2716                 crypt = zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION);
2717                 is_clone = zhp->zfs_dmustats.dds_origin[0] != '\0';
2718                 (void) zfs_crypto_get_encryption_root(zhp, &is_encroot, NULL);
2719
2720                 /* we don't need to do anything for unencrypted filesystems */
2721                 if (crypt == ZIO_CRYPT_OFF) {
2722                         zfs_close(zhp);
2723                         continue;
2724                 }
2725
2726                 /*
2727                  * If the dataset is flagged as an encryption root, was not
2728                  * received as a clone and is not currently an encryption root,
2729                  * force it to become one. Fixup the keylocation if necessary.
2730                  */
2731                 if (stream_encroot) {
2732                         if (!is_clone && !is_encroot) {
2733                                 err = lzc_change_key(fsname,
2734                                     DCP_CMD_FORCE_NEW_KEY, NULL, NULL, 0);
2735                                 if (err != 0) {
2736                                         zfs_close(zhp);
2737                                         goto error;
2738                                 }
2739                         }
2740
2741                         VERIFY(0 == nvlist_lookup_string(props,
2742                             zfs_prop_to_name(ZFS_PROP_KEYLOCATION),
2743                             &stream_keylocation));
2744
2745                         /*
2746                          * Refresh the properties in case the call to
2747                          * lzc_change_key() changed the value.
2748                          */
2749                         zfs_refresh_properties(zhp);
2750                         err = zfs_prop_get(zhp, ZFS_PROP_KEYLOCATION,
2751                             keylocation, sizeof (keylocation), NULL, NULL,
2752                             0, B_TRUE);
2753                         if (err != 0) {
2754                                 zfs_close(zhp);
2755                                 goto error;
2756                         }
2757
2758                         if (strcmp(keylocation, stream_keylocation) != 0) {
2759                                 err = zfs_prop_set(zhp,
2760                                     zfs_prop_to_name(ZFS_PROP_KEYLOCATION),
2761                                     stream_keylocation);
2762                                 if (err != 0) {
2763                                         zfs_close(zhp);
2764                                         goto error;
2765                                 }
2766                         }
2767                 }
2768
2769                 /*
2770                  * If the dataset is not flagged as an encryption root and is
2771                  * currently an encryption root, force it to inherit from its
2772                  * parent. The root of a raw send should never be
2773                  * force-inherited.
2774                  */
2775                 if (!stream_encroot && is_encroot &&
2776                     strcmp(top_zfs, fsname) != 0) {
2777                         err = lzc_change_key(fsname, DCP_CMD_FORCE_INHERIT,
2778                             NULL, NULL, 0);
2779                         if (err != 0) {
2780                                 zfs_close(zhp);
2781                                 goto error;
2782                         }
2783                 }
2784
2785                 zfs_close(zhp);
2786         }
2787
2788         return (0);
2789
2790 error:
2791         return (err);
2792 }
2793
2794 static int
2795 recv_incremental_replication(libzfs_handle_t *hdl, const char *tofs,
2796     recvflags_t *flags, nvlist_t *stream_nv, avl_tree_t *stream_avl,
2797     nvlist_t *renamed)
2798 {
2799         nvlist_t *local_nv, *deleted = NULL;
2800         avl_tree_t *local_avl;
2801         nvpair_t *fselem, *nextfselem;
2802         char *fromsnap;
2803         char newname[ZFS_MAX_DATASET_NAME_LEN];
2804         char guidname[32];
2805         int error;
2806         boolean_t needagain, progress, recursive;
2807         char *s1, *s2;
2808
2809         VERIFY(0 == nvlist_lookup_string(stream_nv, "fromsnap", &fromsnap));
2810
2811         recursive = (nvlist_lookup_boolean(stream_nv, "not_recursive") ==
2812             ENOENT);
2813
2814         if (flags->dryrun)
2815                 return (0);
2816
2817 again:
2818         needagain = progress = B_FALSE;
2819
2820         VERIFY(0 == nvlist_alloc(&deleted, NV_UNIQUE_NAME, 0));
2821
2822         if ((error = gather_nvlist(hdl, tofs, fromsnap, NULL,
2823             recursive, B_TRUE, B_FALSE, B_FALSE, &local_nv, &local_avl)) != 0)
2824                 return (error);
2825
2826         /*
2827          * Process deletes and renames
2828          */
2829         for (fselem = nvlist_next_nvpair(local_nv, NULL);
2830             fselem; fselem = nextfselem) {
2831                 nvlist_t *nvfs, *snaps;
2832                 nvlist_t *stream_nvfs = NULL;
2833                 nvpair_t *snapelem, *nextsnapelem;
2834                 uint64_t fromguid = 0;
2835                 uint64_t originguid = 0;
2836                 uint64_t stream_originguid = 0;
2837                 uint64_t parent_fromsnap_guid, stream_parent_fromsnap_guid;
2838                 char *fsname, *stream_fsname;
2839
2840                 nextfselem = nvlist_next_nvpair(local_nv, fselem);
2841
2842                 VERIFY(0 == nvpair_value_nvlist(fselem, &nvfs));
2843                 VERIFY(0 == nvlist_lookup_nvlist(nvfs, "snaps", &snaps));
2844                 VERIFY(0 == nvlist_lookup_string(nvfs, "name", &fsname));
2845                 VERIFY(0 == nvlist_lookup_uint64(nvfs, "parentfromsnap",
2846                     &parent_fromsnap_guid));
2847                 (void) nvlist_lookup_uint64(nvfs, "origin", &originguid);
2848
2849                 /*
2850                  * First find the stream's fs, so we can check for
2851                  * a different origin (due to "zfs promote")
2852                  */
2853                 for (snapelem = nvlist_next_nvpair(snaps, NULL);
2854                     snapelem; snapelem = nvlist_next_nvpair(snaps, snapelem)) {
2855                         uint64_t thisguid;
2856
2857                         VERIFY(0 == nvpair_value_uint64(snapelem, &thisguid));
2858                         stream_nvfs = fsavl_find(stream_avl, thisguid, NULL);
2859
2860                         if (stream_nvfs != NULL)
2861                                 break;
2862                 }
2863
2864                 /* check for promote */
2865                 (void) nvlist_lookup_uint64(stream_nvfs, "origin",
2866                     &stream_originguid);
2867                 if (stream_nvfs && originguid != stream_originguid) {
2868                         switch (created_before(hdl, local_avl,
2869                             stream_originguid, originguid)) {
2870                         case 1: {
2871                                 /* promote it! */
2872                                 nvlist_t *origin_nvfs;
2873                                 char *origin_fsname;
2874
2875                                 origin_nvfs = fsavl_find(local_avl, originguid,
2876                                     NULL);
2877                                 VERIFY(0 == nvlist_lookup_string(origin_nvfs,
2878                                     "name", &origin_fsname));
2879                                 error = recv_promote(hdl, fsname, origin_fsname,
2880                                     flags);
2881                                 if (error == 0)
2882                                         progress = B_TRUE;
2883                                 break;
2884                         }
2885                         default:
2886                                 break;
2887                         case -1:
2888                                 fsavl_destroy(local_avl);
2889                                 nvlist_free(local_nv);
2890                                 return (-1);
2891                         }
2892                         /*
2893                          * We had/have the wrong origin, therefore our
2894                          * list of snapshots is wrong.  Need to handle
2895                          * them on the next pass.
2896                          */
2897                         needagain = B_TRUE;
2898                         continue;
2899                 }
2900
2901                 for (snapelem = nvlist_next_nvpair(snaps, NULL);
2902                     snapelem; snapelem = nextsnapelem) {
2903                         uint64_t thisguid;
2904                         char *stream_snapname;
2905                         nvlist_t *found, *props;
2906
2907                         nextsnapelem = nvlist_next_nvpair(snaps, snapelem);
2908
2909                         VERIFY(0 == nvpair_value_uint64(snapelem, &thisguid));
2910                         found = fsavl_find(stream_avl, thisguid,
2911                             &stream_snapname);
2912
2913                         /* check for delete */
2914                         if (found == NULL) {
2915                                 char name[ZFS_MAX_DATASET_NAME_LEN];
2916
2917                                 if (!flags->force)
2918                                         continue;
2919
2920                                 (void) snprintf(name, sizeof (name), "%s@%s",
2921                                     fsname, nvpair_name(snapelem));
2922
2923                                 error = recv_destroy(hdl, name,
2924                                     strlen(fsname)+1, newname, flags);
2925                                 if (error)
2926                                         needagain = B_TRUE;
2927                                 else
2928                                         progress = B_TRUE;
2929                                 sprintf(guidname, "%llu",
2930                                     (u_longlong_t)thisguid);
2931                                 nvlist_add_boolean(deleted, guidname);
2932                                 continue;
2933                         }
2934
2935                         stream_nvfs = found;
2936
2937                         if (0 == nvlist_lookup_nvlist(stream_nvfs, "snapprops",
2938                             &props) && 0 == nvlist_lookup_nvlist(props,
2939                             stream_snapname, &props)) {
2940                                 zfs_cmd_t zc = {"\0"};
2941
2942                                 zc.zc_cookie = B_TRUE; /* received */
2943                                 (void) snprintf(zc.zc_name, sizeof (zc.zc_name),
2944                                     "%s@%s", fsname, nvpair_name(snapelem));
2945                                 if (zcmd_write_src_nvlist(hdl, &zc,
2946                                     props) == 0) {
2947                                         (void) zfs_ioctl(hdl,
2948                                             ZFS_IOC_SET_PROP, &zc);
2949                                         zcmd_free_nvlists(&zc);
2950                                 }
2951                         }
2952
2953                         /* check for different snapname */
2954                         if (strcmp(nvpair_name(snapelem),
2955                             stream_snapname) != 0) {
2956                                 char name[ZFS_MAX_DATASET_NAME_LEN];
2957                                 char tryname[ZFS_MAX_DATASET_NAME_LEN];
2958
2959                                 (void) snprintf(name, sizeof (name), "%s@%s",
2960                                     fsname, nvpair_name(snapelem));
2961                                 (void) snprintf(tryname, sizeof (name), "%s@%s",
2962                                     fsname, stream_snapname);
2963
2964                                 error = recv_rename(hdl, name, tryname,
2965                                     strlen(fsname)+1, newname, flags);
2966                                 if (error)
2967                                         needagain = B_TRUE;
2968                                 else
2969                                         progress = B_TRUE;
2970                         }
2971
2972                         if (strcmp(stream_snapname, fromsnap) == 0)
2973                                 fromguid = thisguid;
2974                 }
2975
2976                 /* check for delete */
2977                 if (stream_nvfs == NULL) {
2978                         if (!flags->force)
2979                                 continue;
2980
2981                         error = recv_destroy(hdl, fsname, strlen(tofs)+1,
2982                             newname, flags);
2983                         if (error)
2984                                 needagain = B_TRUE;
2985                         else
2986                                 progress = B_TRUE;
2987                         sprintf(guidname, "%llu",
2988                             (u_longlong_t)parent_fromsnap_guid);
2989                         nvlist_add_boolean(deleted, guidname);
2990                         continue;
2991                 }
2992
2993                 if (fromguid == 0) {
2994                         if (flags->verbose) {
2995                                 (void) printf("local fs %s does not have "
2996                                     "fromsnap (%s in stream); must have "
2997                                     "been deleted locally; ignoring\n",
2998                                     fsname, fromsnap);
2999                         }
3000                         continue;
3001                 }
3002
3003                 VERIFY(0 == nvlist_lookup_string(stream_nvfs,
3004                     "name", &stream_fsname));
3005                 VERIFY(0 == nvlist_lookup_uint64(stream_nvfs,
3006                     "parentfromsnap", &stream_parent_fromsnap_guid));
3007
3008                 s1 = strrchr(fsname, '/');
3009                 s2 = strrchr(stream_fsname, '/');
3010
3011                 /*
3012                  * Check if we're going to rename based on parent guid change
3013                  * and the current parent guid was also deleted. If it was then
3014                  * rename will fail and is likely unneeded, so avoid this and
3015                  * force an early retry to determine the new
3016                  * parent_fromsnap_guid.
3017                  */
3018                 if (stream_parent_fromsnap_guid != 0 &&
3019                     parent_fromsnap_guid != 0 &&
3020                     stream_parent_fromsnap_guid != parent_fromsnap_guid) {
3021                         sprintf(guidname, "%llu",
3022                             (u_longlong_t)parent_fromsnap_guid);
3023                         if (nvlist_exists(deleted, guidname)) {
3024                                 progress = B_TRUE;
3025                                 needagain = B_TRUE;
3026                                 goto doagain;
3027                         }
3028                 }
3029
3030                 /*
3031                  * Check for rename. If the exact receive path is specified, it
3032                  * does not count as a rename, but we still need to check the
3033                  * datasets beneath it.
3034                  */
3035                 if ((stream_parent_fromsnap_guid != 0 &&
3036                     parent_fromsnap_guid != 0 &&
3037                     stream_parent_fromsnap_guid != parent_fromsnap_guid) ||
3038                     ((flags->isprefix || strcmp(tofs, fsname) != 0) &&
3039                     (s1 != NULL) && (s2 != NULL) && strcmp(s1, s2) != 0)) {
3040                         nvlist_t *parent;
3041                         char tryname[ZFS_MAX_DATASET_NAME_LEN];
3042
3043                         parent = fsavl_find(local_avl,
3044                             stream_parent_fromsnap_guid, NULL);
3045                         /*
3046                          * NB: parent might not be found if we used the
3047                          * tosnap for stream_parent_fromsnap_guid,
3048                          * because the parent is a newly-created fs;
3049                          * we'll be able to rename it after we recv the
3050                          * new fs.
3051                          */
3052                         if (parent != NULL) {
3053                                 char *pname;
3054
3055                                 VERIFY(0 == nvlist_lookup_string(parent, "name",
3056                                     &pname));
3057                                 (void) snprintf(tryname, sizeof (tryname),
3058                                     "%s%s", pname, strrchr(stream_fsname, '/'));
3059                         } else {
3060                                 tryname[0] = '\0';
3061                                 if (flags->verbose) {
3062                                         (void) printf("local fs %s new parent "
3063                                             "not found\n", fsname);
3064                                 }
3065                         }
3066
3067                         newname[0] = '\0';
3068
3069                         error = recv_rename(hdl, fsname, tryname,
3070                             strlen(tofs)+1, newname, flags);
3071
3072                         if (renamed != NULL && newname[0] != '\0') {
3073                                 VERIFY(0 == nvlist_add_boolean(renamed,
3074                                     newname));
3075                         }
3076
3077                         if (error)
3078                                 needagain = B_TRUE;
3079                         else
3080                                 progress = B_TRUE;
3081                 }
3082         }
3083
3084 doagain:
3085         fsavl_destroy(local_avl);
3086         nvlist_free(local_nv);
3087         nvlist_free(deleted);
3088
3089         if (needagain && progress) {
3090                 /* do another pass to fix up temporary names */
3091                 if (flags->verbose)
3092                         (void) printf("another pass:\n");
3093                 goto again;
3094         }
3095
3096         return (needagain || error != 0);
3097 }
3098
3099 static int
3100 zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname,
3101     recvflags_t *flags, dmu_replay_record_t *drr, zio_cksum_t *zc,
3102     char **top_zfs, int cleanup_fd, uint64_t *action_handlep,
3103     nvlist_t *cmdprops)
3104 {
3105         nvlist_t *stream_nv = NULL;
3106         avl_tree_t *stream_avl = NULL;
3107         char *fromsnap = NULL;
3108         char *sendsnap = NULL;
3109         char *cp;
3110         char tofs[ZFS_MAX_DATASET_NAME_LEN];
3111         char sendfs[ZFS_MAX_DATASET_NAME_LEN];
3112         char errbuf[1024];
3113         dmu_replay_record_t drre;
3114         int error;
3115         boolean_t anyerr = B_FALSE;
3116         boolean_t softerr = B_FALSE;
3117         boolean_t recursive, raw;
3118
3119         (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
3120             "cannot receive"));
3121
3122         assert(drr->drr_type == DRR_BEGIN);
3123         assert(drr->drr_u.drr_begin.drr_magic == DMU_BACKUP_MAGIC);
3124         assert(DMU_GET_STREAM_HDRTYPE(drr->drr_u.drr_begin.drr_versioninfo) ==
3125             DMU_COMPOUNDSTREAM);
3126
3127         /*
3128          * Read in the nvlist from the stream.
3129          */
3130         if (drr->drr_payloadlen != 0) {
3131                 error = recv_read_nvlist(hdl, fd, drr->drr_payloadlen,
3132                     &stream_nv, flags->byteswap, zc);
3133                 if (error) {
3134                         error = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
3135                         goto out;
3136                 }
3137         }
3138
3139         recursive = (nvlist_lookup_boolean(stream_nv, "not_recursive") ==
3140             ENOENT);
3141         raw = (nvlist_lookup_boolean(stream_nv, "raw") == 0);
3142
3143         if (recursive && strchr(destname, '@')) {
3144                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3145                     "cannot specify snapshot name for multi-snapshot stream"));
3146                 error = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
3147                 goto out;
3148         }
3149
3150         /*
3151          * Read in the end record and verify checksum.
3152          */
3153         if (0 != (error = recv_read(hdl, fd, &drre, sizeof (drre),
3154             flags->byteswap, NULL)))
3155                 goto out;
3156         if (flags->byteswap) {
3157                 drre.drr_type = BSWAP_32(drre.drr_type);
3158                 drre.drr_u.drr_end.drr_checksum.zc_word[0] =
3159                     BSWAP_64(drre.drr_u.drr_end.drr_checksum.zc_word[0]);
3160                 drre.drr_u.drr_end.drr_checksum.zc_word[1] =
3161                     BSWAP_64(drre.drr_u.drr_end.drr_checksum.zc_word[1]);
3162                 drre.drr_u.drr_end.drr_checksum.zc_word[2] =
3163                     BSWAP_64(drre.drr_u.drr_end.drr_checksum.zc_word[2]);
3164                 drre.drr_u.drr_end.drr_checksum.zc_word[3] =
3165                     BSWAP_64(drre.drr_u.drr_end.drr_checksum.zc_word[3]);
3166         }
3167         if (drre.drr_type != DRR_END) {
3168                 error = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
3169                 goto out;
3170         }
3171         if (!ZIO_CHECKSUM_EQUAL(drre.drr_u.drr_end.drr_checksum, *zc)) {
3172                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3173                     "incorrect header checksum"));
3174                 error = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
3175                 goto out;
3176         }
3177
3178         (void) nvlist_lookup_string(stream_nv, "fromsnap", &fromsnap);
3179
3180         if (drr->drr_payloadlen != 0) {
3181                 nvlist_t *stream_fss;
3182
3183                 VERIFY(0 == nvlist_lookup_nvlist(stream_nv, "fss",
3184                     &stream_fss));
3185                 if ((stream_avl = fsavl_create(stream_fss)) == NULL) {
3186                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3187                             "couldn't allocate avl tree"));
3188                         error = zfs_error(hdl, EZFS_NOMEM, errbuf);
3189                         goto out;
3190                 }
3191
3192                 if (fromsnap != NULL && recursive) {
3193                         nvlist_t *renamed = NULL;
3194                         nvpair_t *pair = NULL;
3195
3196                         (void) strlcpy(tofs, destname, sizeof (tofs));
3197                         if (flags->isprefix) {
3198                                 struct drr_begin *drrb = &drr->drr_u.drr_begin;
3199                                 int i;
3200
3201                                 if (flags->istail) {
3202                                         cp = strrchr(drrb->drr_toname, '/');
3203                                         if (cp == NULL) {
3204                                                 (void) strlcat(tofs, "/",
3205                                                     sizeof (tofs));
3206                                                 i = 0;
3207                                         } else {
3208                                                 i = (cp - drrb->drr_toname);
3209                                         }
3210                                 } else {
3211                                         i = strcspn(drrb->drr_toname, "/@");
3212                                 }
3213                                 /* zfs_receive_one() will create_parents() */
3214                                 (void) strlcat(tofs, &drrb->drr_toname[i],
3215                                     sizeof (tofs));
3216                                 *strchr(tofs, '@') = '\0';
3217                         }
3218
3219                         if (!flags->dryrun && !flags->nomount) {
3220                                 VERIFY(0 == nvlist_alloc(&renamed,
3221                                     NV_UNIQUE_NAME, 0));
3222                         }
3223
3224                         softerr = recv_incremental_replication(hdl, tofs, flags,
3225                             stream_nv, stream_avl, renamed);
3226
3227                         /* Unmount renamed filesystems before receiving. */
3228                         while ((pair = nvlist_next_nvpair(renamed,
3229                             pair)) != NULL) {
3230                                 zfs_handle_t *zhp;
3231                                 prop_changelist_t *clp = NULL;
3232
3233                                 zhp = zfs_open(hdl, nvpair_name(pair),
3234                                     ZFS_TYPE_FILESYSTEM);
3235                                 if (zhp != NULL) {
3236                                         clp = changelist_gather(zhp,
3237                                             ZFS_PROP_MOUNTPOINT, 0, 0);
3238                                         zfs_close(zhp);
3239                                         if (clp != NULL) {
3240                                                 softerr |=
3241                                                     changelist_prefix(clp);
3242                                                 changelist_free(clp);
3243                                         }
3244                                 }
3245                         }
3246
3247                         nvlist_free(renamed);
3248                 }
3249         }
3250
3251         /*
3252          * Get the fs specified by the first path in the stream (the top level
3253          * specified by 'zfs send') and pass it to each invocation of
3254          * zfs_receive_one().
3255          */
3256         (void) strlcpy(sendfs, drr->drr_u.drr_begin.drr_toname,
3257             sizeof (sendfs));
3258         if ((cp = strchr(sendfs, '@')) != NULL) {
3259                 *cp = '\0';
3260                 /*
3261                  * Find the "sendsnap", the final snapshot in a replication
3262                  * stream.  zfs_receive_one() handles certain errors
3263                  * differently, depending on if the contained stream is the
3264                  * last one or not.
3265                  */
3266                 sendsnap = (cp + 1);
3267         }
3268
3269         /* Finally, receive each contained stream */
3270         do {
3271                 /*
3272                  * we should figure out if it has a recoverable
3273                  * error, in which case do a recv_skip() and drive on.
3274                  * Note, if we fail due to already having this guid,
3275                  * zfs_receive_one() will take care of it (ie,
3276                  * recv_skip() and return 0).
3277                  */
3278                 error = zfs_receive_impl(hdl, destname, NULL, flags, fd,
3279                     sendfs, stream_nv, stream_avl, top_zfs, cleanup_fd,
3280                     action_handlep, sendsnap, cmdprops);
3281                 if (error == ENODATA) {
3282                         error = 0;
3283                         break;
3284                 }
3285                 anyerr |= error;
3286         } while (error == 0);
3287
3288         if (drr->drr_payloadlen != 0 && recursive && fromsnap != NULL) {
3289                 /*
3290                  * Now that we have the fs's they sent us, try the
3291                  * renames again.
3292                  */
3293                 softerr = recv_incremental_replication(hdl, tofs, flags,
3294                     stream_nv, stream_avl, NULL);
3295         }
3296
3297         if (raw && softerr == 0) {
3298                 softerr = recv_fix_encryption_heirarchy(hdl, destname,
3299                     stream_nv, stream_avl);
3300         }
3301
3302 out:
3303         fsavl_destroy(stream_avl);
3304         nvlist_free(stream_nv);
3305         if (softerr)
3306                 error = -2;
3307         if (anyerr)
3308                 error = -1;
3309         return (error);
3310 }
3311
3312 static void
3313 trunc_prop_errs(int truncated)
3314 {
3315         ASSERT(truncated != 0);
3316
3317         if (truncated == 1)
3318                 (void) fprintf(stderr, dgettext(TEXT_DOMAIN,
3319                     "1 more property could not be set\n"));
3320         else
3321                 (void) fprintf(stderr, dgettext(TEXT_DOMAIN,
3322                     "%d more properties could not be set\n"), truncated);
3323 }
3324
3325 static int
3326 recv_skip(libzfs_handle_t *hdl, int fd, boolean_t byteswap)
3327 {
3328         dmu_replay_record_t *drr;
3329         void *buf = zfs_alloc(hdl, SPA_MAXBLOCKSIZE);
3330         char errbuf[1024];
3331
3332         (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
3333             "cannot receive:"));
3334
3335         /* XXX would be great to use lseek if possible... */
3336         drr = buf;
3337
3338         while (recv_read(hdl, fd, drr, sizeof (dmu_replay_record_t),
3339             byteswap, NULL) == 0) {
3340                 if (byteswap)
3341                         drr->drr_type = BSWAP_32(drr->drr_type);
3342
3343                 switch (drr->drr_type) {
3344                 case DRR_BEGIN:
3345                         if (drr->drr_payloadlen != 0) {
3346                                 (void) recv_read(hdl, fd, buf,
3347                                     drr->drr_payloadlen, B_FALSE, NULL);
3348                         }
3349                         break;
3350
3351                 case DRR_END:
3352                         free(buf);
3353                         return (0);
3354
3355                 case DRR_OBJECT:
3356                         if (byteswap) {
3357                                 drr->drr_u.drr_object.drr_bonuslen =
3358                                     BSWAP_32(drr->drr_u.drr_object.
3359                                     drr_bonuslen);
3360                         }
3361                         (void) recv_read(hdl, fd, buf,
3362                             P2ROUNDUP(drr->drr_u.drr_object.drr_bonuslen, 8),
3363                             B_FALSE, NULL);
3364                         break;
3365
3366                 case DRR_WRITE:
3367                         if (byteswap) {
3368                                 drr->drr_u.drr_write.drr_logical_size =
3369                                     BSWAP_64(
3370                                     drr->drr_u.drr_write.drr_logical_size);
3371                                 drr->drr_u.drr_write.drr_compressed_size =
3372                                     BSWAP_64(
3373                                     drr->drr_u.drr_write.drr_compressed_size);
3374                         }
3375                         uint64_t payload_size =
3376                             DRR_WRITE_PAYLOAD_SIZE(&drr->drr_u.drr_write);
3377                         (void) recv_read(hdl, fd, buf,
3378                             payload_size, B_FALSE, NULL);
3379                         break;
3380                 case DRR_SPILL:
3381                         if (byteswap) {
3382                                 drr->drr_u.drr_spill.drr_length =
3383                                     BSWAP_64(drr->drr_u.drr_spill.drr_length);
3384                         }
3385                         (void) recv_read(hdl, fd, buf,
3386                             drr->drr_u.drr_spill.drr_length, B_FALSE, NULL);
3387                         break;
3388                 case DRR_WRITE_EMBEDDED:
3389                         if (byteswap) {
3390                                 drr->drr_u.drr_write_embedded.drr_psize =
3391                                     BSWAP_32(drr->drr_u.drr_write_embedded.
3392                                     drr_psize);
3393                         }
3394                         (void) recv_read(hdl, fd, buf,
3395                             P2ROUNDUP(drr->drr_u.drr_write_embedded.drr_psize,
3396                             8), B_FALSE, NULL);
3397                         break;
3398                 case DRR_WRITE_BYREF:
3399                 case DRR_FREEOBJECTS:
3400                 case DRR_FREE:
3401                         break;
3402
3403                 default:
3404                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3405                             "invalid record type"));
3406                         free(buf);
3407                         return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
3408                 }
3409         }
3410
3411         free(buf);
3412         return (-1);
3413 }
3414
3415 static void
3416 recv_ecksum_set_aux(libzfs_handle_t *hdl, const char *target_snap,
3417     boolean_t resumable)
3418 {
3419         char target_fs[ZFS_MAX_DATASET_NAME_LEN];
3420
3421         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3422             "checksum mismatch or incomplete stream"));
3423
3424         if (!resumable)
3425                 return;
3426         (void) strlcpy(target_fs, target_snap, sizeof (target_fs));
3427         *strchr(target_fs, '@') = '\0';
3428         zfs_handle_t *zhp = zfs_open(hdl, target_fs,
3429             ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
3430         if (zhp == NULL)
3431                 return;
3432
3433         char token_buf[ZFS_MAXPROPLEN];
3434         int error = zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN,
3435             token_buf, sizeof (token_buf),
3436             NULL, NULL, 0, B_TRUE);
3437         if (error == 0) {
3438                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3439                     "checksum mismatch or incomplete stream.\n"
3440                     "Partially received snapshot is saved.\n"
3441                     "A resuming stream can be generated on the sending "
3442                     "system by running:\n"
3443                     "    zfs send -t %s"),
3444                     token_buf);
3445         }
3446         zfs_close(zhp);
3447 }
3448
3449 /*
3450  * Prepare a new nvlist of properties that are to override (-o) or be excluded
3451  * (-x) from the received dataset
3452  * recvprops: received properties from the send stream
3453  * cmdprops: raw input properties from command line
3454  * origprops: properties, both locally-set and received, currently set on the
3455  *            target dataset if it exists, NULL otherwise.
3456  * oxprops: valid output override (-o) and excluded (-x) properties
3457  */
3458 static int
3459 zfs_setup_cmdline_props(libzfs_handle_t *hdl, zfs_type_t type,
3460     char *fsname, boolean_t zoned, boolean_t recursive, boolean_t newfs,
3461     boolean_t raw, boolean_t toplevel, nvlist_t *recvprops, nvlist_t *cmdprops,
3462     nvlist_t *origprops, nvlist_t **oxprops, uint8_t **wkeydata_out,
3463     uint_t *wkeylen_out, const char *errbuf)
3464 {
3465         nvpair_t *nvp;
3466         nvlist_t *oprops, *voprops;
3467         zfs_handle_t *zhp = NULL;
3468         zpool_handle_t *zpool_hdl = NULL;
3469         char *cp;
3470         int ret = 0;
3471         char namebuf[ZFS_MAX_DATASET_NAME_LEN];
3472
3473         if (nvlist_empty(cmdprops))
3474                 return (0); /* No properties to override or exclude */
3475
3476         *oxprops = fnvlist_alloc();
3477         oprops = fnvlist_alloc();
3478
3479         strlcpy(namebuf, fsname, ZFS_MAX_DATASET_NAME_LEN);
3480
3481         /*
3482          * Get our dataset handle. The target dataset may not exist yet.
3483          */
3484         if (zfs_dataset_exists(hdl, namebuf, ZFS_TYPE_DATASET)) {
3485                 zhp = zfs_open(hdl, namebuf, ZFS_TYPE_DATASET);
3486                 if (zhp == NULL) {
3487                         ret = -1;
3488                         goto error;
3489                 }
3490         }
3491
3492         /* open the zpool handle */
3493         cp = strchr(namebuf, '/');
3494         if (cp != NULL)
3495                 *cp = '\0';
3496         zpool_hdl = zpool_open(hdl, namebuf);
3497         if (zpool_hdl == NULL) {
3498                 ret = -1;
3499                 goto error;
3500         }
3501
3502         /* restore namebuf to match fsname for later use */
3503         if (cp != NULL)
3504                 *cp = '/';
3505
3506         /*
3507          * first iteration: process excluded (-x) properties now and gather
3508          * added (-o) properties to be later processed by zfs_valid_proplist()
3509          */
3510         nvp = NULL;
3511         while ((nvp = nvlist_next_nvpair(cmdprops, nvp)) != NULL) {
3512                 const char *name = nvpair_name(nvp);
3513                 zfs_prop_t prop = zfs_name_to_prop(name);
3514
3515                 /* "origin" is processed separately, don't handle it here */
3516                 if (prop == ZFS_PROP_ORIGIN)
3517                         continue;
3518
3519                 /*
3520                  * we're trying to override or exclude a property that does not
3521                  * make sense for this type of dataset, but we don't want to
3522                  * fail if the receive is recursive: this comes in handy when
3523                  * the send stream contains, for instance, a child ZVOL and
3524                  * we're trying to receive it with "-o atime=on"
3525                  */
3526                 if (!zfs_prop_valid_for_type(prop, type, B_FALSE) &&
3527                     !zfs_prop_user(name)) {
3528                         if (recursive)
3529                                 continue;
3530                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3531                             "property '%s' does not apply to datasets of this "
3532                             "type"), name);
3533                         ret = zfs_error(hdl, EZFS_BADPROP, errbuf);
3534                         goto error;
3535                 }
3536
3537                 /* raw streams can't override encryption properties */
3538                 if ((zfs_prop_encryption_key_param(prop) ||
3539                     prop == ZFS_PROP_ENCRYPTION) && (raw || !newfs)) {
3540                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3541                             "encryption property '%s' cannot "
3542                             "be set or excluded for raw or incremental "
3543                             "streams."), name);
3544                         ret = zfs_error(hdl, EZFS_BADPROP, errbuf);
3545                         goto error;
3546                 }
3547
3548                 switch (nvpair_type(nvp)) {
3549                 case DATA_TYPE_BOOLEAN: /* -x property */
3550                         /*
3551                          * DATA_TYPE_BOOLEAN is the way we're asked to "exclude"
3552                          * a property: this is done by forcing an explicit
3553                          * inherit on the destination so the effective value is
3554                          * not the one we received from the send stream.
3555                          * We do this only if the property is not already
3556                          * locally-set, in which case its value will take
3557                          * priority over the received anyway.
3558                          */
3559                         if (nvlist_exists(origprops, name)) {
3560                                 nvlist_t *attrs;
3561
3562                                 attrs = fnvlist_lookup_nvlist(origprops, name);
3563                                 if (strcmp(fnvlist_lookup_string(attrs,
3564                                     ZPROP_SOURCE), ZPROP_SOURCE_VAL_RECVD) != 0)
3565                                         continue;
3566                         }
3567                         /*
3568                          * We can't force an explicit inherit on non-inheritable
3569                          * properties: if we're asked to exclude this kind of
3570                          * values we remove them from "recvprops" input nvlist.
3571                          */
3572                         if (!zfs_prop_inheritable(prop) &&
3573                             !zfs_prop_user(name) && /* can be inherited too */
3574                             nvlist_exists(recvprops, name))
3575                                 fnvlist_remove(recvprops, name);
3576                         else
3577                                 fnvlist_add_nvpair(*oxprops, nvp);
3578                         break;
3579                 case DATA_TYPE_STRING: /* -o property=value */
3580                         fnvlist_add_nvpair(oprops, nvp);
3581                         break;
3582                 default:
3583                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3584                             "property '%s' must be a string or boolean"), name);
3585                         ret = zfs_error(hdl, EZFS_BADPROP, errbuf);
3586                         goto error;
3587                 }
3588         }
3589
3590         if (toplevel) {
3591                 /* convert override strings properties to native */
3592                 if ((voprops = zfs_valid_proplist(hdl, ZFS_TYPE_DATASET,
3593                     oprops, zoned, zhp, zpool_hdl, B_FALSE, errbuf)) == NULL) {
3594                         ret = zfs_error(hdl, EZFS_BADPROP, errbuf);
3595                         goto error;
3596                 }
3597
3598                 /*
3599                  * zfs_crypto_create() requires the parent name. Get it
3600                  * by truncating the fsname copy stored in namebuf.
3601                  */
3602                 cp = strrchr(namebuf, '/');
3603                 if (cp != NULL)
3604                         *cp = '\0';
3605
3606                 if (!raw && zfs_crypto_create(hdl, namebuf, voprops, NULL,
3607                     B_FALSE, wkeydata_out, wkeylen_out) != 0) {
3608                         fnvlist_free(voprops);
3609                         ret = zfs_error(hdl, EZFS_CRYPTOFAILED, errbuf);
3610                         goto error;
3611                 }
3612
3613                 /* second pass: process "-o" properties */
3614                 fnvlist_merge(*oxprops, voprops);
3615                 fnvlist_free(voprops);
3616         } else {
3617                 /* override props on child dataset are inherited */
3618                 nvp = NULL;
3619                 while ((nvp = nvlist_next_nvpair(oprops, nvp)) != NULL) {
3620                         const char *name = nvpair_name(nvp);
3621                         fnvlist_add_boolean(*oxprops, name);
3622                 }
3623         }
3624
3625 error:
3626         if (zhp != NULL)
3627                 zfs_close(zhp);
3628         if (zpool_hdl != NULL)
3629                 zpool_close(zpool_hdl);
3630         fnvlist_free(oprops);
3631         return (ret);
3632 }
3633
3634 /*
3635  * Restores a backup of tosnap from the file descriptor specified by infd.
3636  */
3637 static int
3638 zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
3639     const char *originsnap, recvflags_t *flags, dmu_replay_record_t *drr,
3640     dmu_replay_record_t *drr_noswap, const char *sendfs, nvlist_t *stream_nv,
3641     avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd,
3642     uint64_t *action_handlep, const char *finalsnap, nvlist_t *cmdprops)
3643 {
3644         time_t begin_time;
3645         int ioctl_err, ioctl_errno, err;
3646         char *cp;
3647         struct drr_begin *drrb = &drr->drr_u.drr_begin;
3648         char errbuf[1024];
3649         const char *chopprefix;
3650         boolean_t newfs = B_FALSE;
3651         boolean_t stream_wantsnewfs;
3652         boolean_t newprops = B_FALSE;
3653         uint64_t read_bytes = 0;
3654         uint64_t errflags = 0;
3655         uint64_t parent_snapguid = 0;
3656         prop_changelist_t *clp = NULL;
3657         nvlist_t *snapprops_nvlist = NULL;
3658         zprop_errflags_t prop_errflags;
3659         nvlist_t *prop_errors = NULL;
3660         boolean_t recursive;
3661         char *snapname = NULL;
3662         char destsnap[MAXPATHLEN * 2];
3663         char origin[MAXNAMELEN];
3664         char name[MAXPATHLEN];
3665         char tmp_keylocation[MAXNAMELEN];
3666         nvlist_t *rcvprops = NULL; /* props received from the send stream */
3667         nvlist_t *oxprops = NULL; /* override (-o) and exclude (-x) props */
3668         nvlist_t *origprops = NULL; /* original props (if destination exists) */
3669         zfs_type_t type;
3670         boolean_t toplevel = B_FALSE;
3671         boolean_t zoned = B_FALSE;
3672         boolean_t hastoken = B_FALSE;
3673         uint8_t *wkeydata = NULL;
3674         uint_t wkeylen = 0;
3675
3676         begin_time = time(NULL);
3677         bzero(origin, MAXNAMELEN);
3678         bzero(tmp_keylocation, MAXNAMELEN);
3679
3680         (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
3681             "cannot receive"));
3682
3683         recursive = (nvlist_lookup_boolean(stream_nv, "not_recursive") ==
3684             ENOENT);
3685
3686         if (stream_avl != NULL) {
3687                 char *keylocation = NULL;
3688                 nvlist_t *lookup = NULL;
3689                 nvlist_t *fs = fsavl_find(stream_avl, drrb->drr_toguid,
3690                     &snapname);
3691
3692                 (void) nvlist_lookup_uint64(fs, "parentfromsnap",
3693                     &parent_snapguid);
3694                 err = nvlist_lookup_nvlist(fs, "props", &rcvprops);
3695                 if (err) {
3696                         VERIFY(0 == nvlist_alloc(&rcvprops, NV_UNIQUE_NAME, 0));
3697                         newprops = B_TRUE;
3698                 }
3699
3700                 /*
3701                  * The keylocation property may only be set on encryption roots,
3702                  * but this dataset might not become an encryption root until
3703                  * recv_fix_encryption_heirarchy() is called. That function
3704                  * will fixup the keylocation anyway, so we temporarily unset
3705                  * the keylocation for now to avoid any errors from the receive
3706                  * ioctl.
3707                  */
3708                 err = nvlist_lookup_string(rcvprops,
3709                     zfs_prop_to_name(ZFS_PROP_KEYLOCATION), &keylocation);
3710                 if (err == 0) {
3711                         strcpy(tmp_keylocation, keylocation);
3712                         (void) nvlist_remove_all(rcvprops,
3713                             zfs_prop_to_name(ZFS_PROP_KEYLOCATION));
3714                 }
3715
3716                 if (flags->canmountoff) {
3717                         VERIFY(0 == nvlist_add_uint64(rcvprops,
3718                             zfs_prop_to_name(ZFS_PROP_CANMOUNT), 0));
3719                 }
3720                 if (0 == nvlist_lookup_nvlist(fs, "snapprops", &lookup)) {
3721                         VERIFY(0 == nvlist_lookup_nvlist(lookup,
3722                             snapname, &snapprops_nvlist));
3723                 }
3724         }
3725
3726         cp = NULL;
3727
3728         /*
3729          * Determine how much of the snapshot name stored in the stream
3730          * we are going to tack on to the name they specified on the
3731          * command line, and how much we are going to chop off.
3732          *
3733          * If they specified a snapshot, chop the entire name stored in
3734          * the stream.
3735          */
3736         if (flags->istail) {
3737                 /*
3738                  * A filesystem was specified with -e. We want to tack on only
3739                  * the tail of the sent snapshot path.
3740                  */
3741                 if (strchr(tosnap, '@')) {
3742                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
3743                             "argument - snapshot not allowed with -e"));
3744                         err = zfs_error(hdl, EZFS_INVALIDNAME, errbuf);
3745                         goto out;
3746                 }
3747
3748                 chopprefix = strrchr(sendfs, '/');
3749
3750                 if (chopprefix == NULL) {
3751                         /*
3752                          * The tail is the poolname, so we need to
3753                          * prepend a path separator.
3754                          */
3755                         int len = strlen(drrb->drr_toname);
3756                         cp = malloc(len + 2);
3757                         cp[0] = '/';
3758                         (void) strcpy(&cp[1], drrb->drr_toname);
3759                         chopprefix = cp;
3760                 } else {
3761                         chopprefix = drrb->drr_toname + (chopprefix - sendfs);
3762                 }
3763         } else if (flags->isprefix) {
3764                 /*
3765                  * A filesystem was specified with -d. We want to tack on
3766                  * everything but the first element of the sent snapshot path
3767                  * (all but the pool name).
3768                  */
3769                 if (strchr(tosnap, '@')) {
3770                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
3771                             "argument - snapshot not allowed with -d"));
3772                         err = zfs_error(hdl, EZFS_INVALIDNAME, errbuf);
3773                         goto out;
3774                 }
3775
3776                 chopprefix = strchr(drrb->drr_toname, '/');
3777                 if (chopprefix == NULL)
3778                         chopprefix = strchr(drrb->drr_toname, '@');
3779         } else if (strchr(tosnap, '@') == NULL) {
3780                 /*
3781                  * If a filesystem was specified without -d or -e, we want to
3782                  * tack on everything after the fs specified by 'zfs send'.
3783                  */
3784                 chopprefix = drrb->drr_toname + strlen(sendfs);
3785         } else {
3786                 /* A snapshot was specified as an exact path (no -d or -e). */
3787                 if (recursive) {
3788                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3789                             "cannot specify snapshot name for multi-snapshot "
3790                             "stream"));
3791                         err = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
3792                         goto out;
3793                 }
3794                 chopprefix = drrb->drr_toname + strlen(drrb->drr_toname);
3795         }
3796
3797         ASSERT(strstr(drrb->drr_toname, sendfs) == drrb->drr_toname);
3798         ASSERT(chopprefix > drrb->drr_toname || strchr(sendfs, '/') == NULL);
3799         ASSERT(chopprefix <= drrb->drr_toname + strlen(drrb->drr_toname) ||
3800             strchr(sendfs, '/') == NULL);
3801         ASSERT(chopprefix[0] == '/' || chopprefix[0] == '@' ||
3802             chopprefix[0] == '\0');
3803
3804         /*
3805          * Determine name of destination snapshot.
3806          */
3807         (void) strlcpy(destsnap, tosnap, sizeof (destsnap));
3808         (void) strlcat(destsnap, chopprefix, sizeof (destsnap));
3809         free(cp);
3810         if (!zfs_name_valid(destsnap, ZFS_TYPE_SNAPSHOT)) {
3811                 err = zfs_error(hdl, EZFS_INVALIDNAME, errbuf);
3812                 goto out;
3813         }
3814
3815         /*
3816          * Determine the name of the origin snapshot.
3817          */
3818         if (originsnap) {
3819                 (void) strlcpy(origin, originsnap, sizeof (origin));
3820                 if (flags->verbose)
3821                         (void) printf("using provided clone origin %s\n",
3822                             origin);
3823         } else if (drrb->drr_flags & DRR_FLAG_CLONE) {
3824                 if (guid_to_name(hdl, destsnap,
3825                     drrb->drr_fromguid, B_FALSE, origin) != 0) {
3826                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3827                             "local origin for clone %s does not exist"),
3828                             destsnap);
3829                         err = zfs_error(hdl, EZFS_NOENT, errbuf);
3830                         goto out;
3831                 }
3832                 if (flags->verbose)
3833                         (void) printf("found clone origin %s\n", origin);
3834         }
3835
3836         boolean_t resuming = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
3837             DMU_BACKUP_FEATURE_RESUMING;
3838         boolean_t raw = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
3839             DMU_BACKUP_FEATURE_RAW;
3840         boolean_t embedded = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
3841             DMU_BACKUP_FEATURE_EMBED_DATA;
3842         stream_wantsnewfs = (drrb->drr_fromguid == 0 ||
3843             (drrb->drr_flags & DRR_FLAG_CLONE) || originsnap) && !resuming;
3844
3845         if (stream_wantsnewfs) {
3846                 /*
3847                  * if the parent fs does not exist, look for it based on
3848                  * the parent snap GUID
3849                  */
3850                 (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
3851                     "cannot receive new filesystem stream"));
3852
3853                 (void) strcpy(name, destsnap);
3854                 cp = strrchr(name, '/');
3855                 if (cp)
3856                         *cp = '\0';
3857                 if (cp &&
3858                     !zfs_dataset_exists(hdl, name, ZFS_TYPE_DATASET)) {
3859                         char suffix[ZFS_MAX_DATASET_NAME_LEN];
3860                         (void) strcpy(suffix, strrchr(destsnap, '/'));
3861                         if (guid_to_name(hdl, name, parent_snapguid,
3862                             B_FALSE, destsnap) == 0) {
3863                                 *strchr(destsnap, '@') = '\0';
3864                                 (void) strcat(destsnap, suffix);
3865                         }
3866                 }
3867         } else {
3868                 /*
3869                  * if the fs does not exist, look for it based on the
3870                  * fromsnap GUID
3871                  */
3872                 (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
3873                     "cannot receive incremental stream"));
3874
3875                 (void) strcpy(name, destsnap);
3876                 *strchr(name, '@') = '\0';
3877
3878                 /*
3879                  * If the exact receive path was specified and this is the
3880                  * topmost path in the stream, then if the fs does not exist we
3881                  * should look no further.
3882                  */
3883                 if ((flags->isprefix || (*(chopprefix = drrb->drr_toname +
3884                     strlen(sendfs)) != '\0' && *chopprefix != '@')) &&
3885                     !zfs_dataset_exists(hdl, name, ZFS_TYPE_DATASET)) {
3886                         char snap[ZFS_MAX_DATASET_NAME_LEN];
3887                         (void) strcpy(snap, strchr(destsnap, '@'));
3888                         if (guid_to_name(hdl, name, drrb->drr_fromguid,
3889                             B_FALSE, destsnap) == 0) {
3890                                 *strchr(destsnap, '@') = '\0';
3891                                 (void) strcat(destsnap, snap);
3892                         }
3893                 }
3894         }
3895
3896         (void) strcpy(name, destsnap);
3897         *strchr(name, '@') = '\0';
3898
3899         if (zfs_dataset_exists(hdl, name, ZFS_TYPE_DATASET)) {
3900                 zfs_cmd_t zc = {"\0"};
3901                 zfs_handle_t *zhp;
3902                 boolean_t encrypted;
3903
3904                 (void) strcpy(zc.zc_name, name);
3905
3906                 /*
3907                  * Destination fs exists.  It must be one of these cases:
3908                  *  - an incremental send stream
3909                  *  - the stream specifies a new fs (full stream or clone)
3910                  *    and they want us to blow away the existing fs (and
3911                  *    have therefore specified -F and removed any snapshots)
3912                  *  - we are resuming a failed receive.
3913                  */
3914                 if (stream_wantsnewfs) {
3915                         boolean_t is_volume = drrb->drr_type == DMU_OST_ZVOL;
3916                         if (!flags->force) {
3917                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3918                                     "destination '%s' exists\n"
3919                                     "must specify -F to overwrite it"), name);
3920                                 err = zfs_error(hdl, EZFS_EXISTS, errbuf);
3921                                 goto out;
3922                         }
3923                         if (ioctl(hdl->libzfs_fd, ZFS_IOC_SNAPSHOT_LIST_NEXT,
3924                             &zc) == 0) {
3925                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3926                                     "destination has snapshots (eg. %s)\n"
3927                                     "must destroy them to overwrite it"),
3928                                     zc.zc_name);
3929                                 err = zfs_error(hdl, EZFS_EXISTS, errbuf);
3930                                 goto out;
3931                         }
3932                         if (is_volume && strrchr(name, '/') == NULL) {
3933                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3934                                     "destination %s is the root dataset\n"
3935                                     "cannot overwrite with a ZVOL"),
3936                                     name);
3937                                 err = zfs_error(hdl, EZFS_EXISTS, errbuf);
3938                                 goto out;
3939                         }
3940                         if (is_volume &&
3941                             ioctl(hdl->libzfs_fd, ZFS_IOC_DATASET_LIST_NEXT,
3942                             &zc) == 0) {
3943                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3944                                     "destination has children (eg. %s)\n"
3945                                     "cannot overwrite with a ZVOL"),
3946                                     zc.zc_name);
3947                                 err = zfs_error(hdl, EZFS_WRONG_PARENT, errbuf);
3948                                 goto out;
3949                         }
3950                 }
3951
3952                 if ((zhp = zfs_open(hdl, name,
3953                     ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME)) == NULL) {
3954                         err = -1;
3955                         goto out;
3956                 }
3957
3958                 if (stream_wantsnewfs &&
3959                     zhp->zfs_dmustats.dds_origin[0]) {
3960                         zfs_close(zhp);
3961                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3962                             "destination '%s' is a clone\n"
3963                             "must destroy it to overwrite it"), name);
3964                         err = zfs_error(hdl, EZFS_EXISTS, errbuf);
3965                         goto out;
3966                 }
3967
3968                 /*
3969                  * Raw sends can not be performed as an incremental on top
3970                  * of existing unencryppted datasets. zfs recv -F cant be
3971                  * used to blow away an existing encrypted filesystem. This
3972                  * is because it would require the dsl dir to point to the
3973                  * new key (or lack of a key) and the old key at the same
3974                  * time. The -F flag may still be used for deleting
3975                  * intermediate snapshots that would otherwise prevent the
3976                  * receive from working.
3977                  */
3978                 encrypted = zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION) !=
3979                     ZIO_CRYPT_OFF;
3980                 if (!stream_wantsnewfs && !encrypted && raw) {
3981                         zfs_close(zhp);
3982                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3983                             "cannot perform raw receive on top of "
3984                             "existing unencrypted dataset"));
3985                         err = zfs_error(hdl, EZFS_BADRESTORE, errbuf);
3986                         goto out;
3987                 }
3988
3989                 if (stream_wantsnewfs && flags->force &&
3990                     ((raw && !encrypted) || encrypted)) {
3991                         zfs_close(zhp);
3992                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
3993                             "zfs receive -F cannot be used to destroy an "
3994                             "encrypted filesystem or overwrite an "
3995                             "unencrypted one with an encrypted one"));
3996                         err = zfs_error(hdl, EZFS_BADRESTORE, errbuf);
3997                         goto out;
3998                 }
3999
4000                 if (!flags->dryrun && zhp->zfs_type == ZFS_TYPE_FILESYSTEM &&
4001                     stream_wantsnewfs) {
4002                         /* We can't do online recv in this case */
4003                         clp = changelist_gather(zhp, ZFS_PROP_NAME, 0, 0);
4004                         if (clp == NULL) {
4005                                 zfs_close(zhp);
4006                                 err = -1;
4007                                 goto out;
4008                         }
4009                         if (changelist_prefix(clp) != 0) {
4010                                 changelist_free(clp);
4011                                 zfs_close(zhp);
4012                                 err = -1;
4013                                 goto out;
4014                         }
4015                 }
4016
4017                 /*
4018                  * If we are resuming a newfs, set newfs here so that we will
4019                  * mount it if the recv succeeds this time.  We can tell
4020                  * that it was a newfs on the first recv because the fs
4021                  * itself will be inconsistent (if the fs existed when we
4022                  * did the first recv, we would have received it into
4023                  * .../%recv).
4024                  */
4025                 if (resuming && zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT))
4026                         newfs = B_TRUE;
4027
4028                 /* we want to know if we're zoned when validating -o|-x props */
4029                 zoned = zfs_prop_get_int(zhp, ZFS_PROP_ZONED);
4030
4031                 /* may need this info later, get it now we have zhp around */
4032                 if (zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN, NULL, 0,
4033                     NULL, NULL, 0, B_TRUE) == 0)
4034                         hastoken = B_TRUE;
4035
4036                 /* gather existing properties on destination */
4037                 origprops = fnvlist_alloc();
4038                 fnvlist_merge(origprops, zhp->zfs_props);
4039                 fnvlist_merge(origprops, zhp->zfs_user_props);
4040
4041                 zfs_close(zhp);
4042         } else {
4043                 zfs_handle_t *zhp;
4044
4045                 /*
4046                  * Destination filesystem does not exist.  Therefore we better
4047                  * be creating a new filesystem (either from a full backup, or
4048                  * a clone).  It would therefore be invalid if the user
4049                  * specified only the pool name (i.e. if the destination name
4050                  * contained no slash character).
4051                  */
4052                 cp = strrchr(name, '/');
4053
4054                 if (!stream_wantsnewfs || cp == NULL) {
4055                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4056                             "destination '%s' does not exist"), name);
4057                         err = zfs_error(hdl, EZFS_NOENT, errbuf);
4058                         goto out;
4059                 }
4060
4061                 /*
4062                  * Trim off the final dataset component so we perform the
4063                  * recvbackup ioctl to the filesystems's parent.
4064                  */
4065                 *cp = '\0';
4066
4067                 if (flags->isprefix && !flags->istail && !flags->dryrun &&
4068                     create_parents(hdl, destsnap, strlen(tosnap)) != 0) {
4069                         err = zfs_error(hdl, EZFS_BADRESTORE, errbuf);
4070                         goto out;
4071                 }
4072
4073                 /* validate parent */
4074                 zhp = zfs_open(hdl, name, ZFS_TYPE_DATASET);
4075                 if (zhp == NULL) {
4076                         err = zfs_error(hdl, EZFS_BADRESTORE, errbuf);
4077                         goto out;
4078                 }
4079                 if (zfs_get_type(zhp) != ZFS_TYPE_FILESYSTEM) {
4080                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4081                             "parent '%s' is not a filesystem"), name);
4082                         err = zfs_error(hdl, EZFS_WRONG_PARENT, errbuf);
4083                         zfs_close(zhp);
4084                         goto out;
4085                 }
4086
4087                 /*
4088                  * It is invalid to receive a properties stream that was
4089                  * unencrypted on the send side as a child of an encrypted
4090                  * parent. Technically there is nothing preventing this, but
4091                  * it would mean that the encryption=off property which is
4092                  * locally set on the send side would not be received correctly.
4093                  * We can infer encryption=off if the stream is not raw and
4094                  * properties were included since the send side will only ever
4095                  * send the encryption property in a raw nvlist header. This
4096                  * check will be avoided if the user specifically overrides
4097                  * the encryption property on the command line.
4098                  */
4099                 if (!raw && rcvprops != NULL &&
4100                     !nvlist_exists(cmdprops,
4101                     zfs_prop_to_name(ZFS_PROP_ENCRYPTION))) {
4102                         uint64_t crypt;
4103
4104                         crypt = zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION);
4105
4106                         if (crypt != ZIO_CRYPT_OFF) {
4107                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4108                                     "parent '%s' must not be encrypted to "
4109                                     "receive unenecrypted property"), name);
4110                                 err = zfs_error(hdl, EZFS_BADPROP, errbuf);
4111                                 zfs_close(zhp);
4112                                 goto out;
4113                         }
4114                 }
4115                 zfs_close(zhp);
4116
4117                 newfs = B_TRUE;
4118                 *cp = '/';
4119         }
4120
4121         if (flags->verbose) {
4122                 (void) printf("%s %s stream of %s into %s\n",
4123                     flags->dryrun ? "would receive" : "receiving",
4124                     drrb->drr_fromguid ? "incremental" : "full",
4125                     drrb->drr_toname, destsnap);
4126                 (void) fflush(stdout);
4127         }
4128
4129         if (flags->dryrun) {
4130                 err = recv_skip(hdl, infd, flags->byteswap);
4131                 goto out;
4132         }
4133
4134         if (top_zfs && (*top_zfs == NULL || strcmp(*top_zfs, name) == 0))
4135                 toplevel = B_TRUE;
4136         if (drrb->drr_type == DMU_OST_ZVOL) {
4137                 type = ZFS_TYPE_VOLUME;
4138         } else if (drrb->drr_type == DMU_OST_ZFS) {
4139                 type = ZFS_TYPE_FILESYSTEM;
4140         } else {
4141                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4142                     "invalid record type: 0x%d"), drrb->drr_type);
4143                 err = zfs_error(hdl, EZFS_BADSTREAM, errbuf);
4144                 goto out;
4145         }
4146         if ((err = zfs_setup_cmdline_props(hdl, type, name, zoned, recursive,
4147             stream_wantsnewfs, raw, toplevel, rcvprops, cmdprops, origprops,
4148             &oxprops, &wkeydata, &wkeylen, errbuf)) != 0)
4149                 goto out;
4150
4151         err = ioctl_err = lzc_receive_with_cmdprops(destsnap, rcvprops,
4152             oxprops, wkeydata, wkeylen, origin, flags->force, flags->resumable,
4153             raw, infd, drr_noswap, cleanup_fd, &read_bytes, &errflags,
4154             action_handlep, &prop_errors);
4155         ioctl_errno = ioctl_err;
4156         prop_errflags = errflags;
4157
4158         if (err == 0) {
4159                 nvpair_t *prop_err = NULL;
4160
4161                 while ((prop_err = nvlist_next_nvpair(prop_errors,
4162                     prop_err)) != NULL) {
4163                         char tbuf[1024];
4164                         zfs_prop_t prop;
4165                         int intval;
4166
4167                         prop = zfs_name_to_prop(nvpair_name(prop_err));
4168                         (void) nvpair_value_int32(prop_err, &intval);
4169                         if (strcmp(nvpair_name(prop_err),
4170                             ZPROP_N_MORE_ERRORS) == 0) {
4171                                 trunc_prop_errs(intval);
4172                                 break;
4173                         } else if (snapname == NULL || finalsnap == NULL ||
4174                             strcmp(finalsnap, snapname) == 0 ||
4175                             strcmp(nvpair_name(prop_err),
4176                             zfs_prop_to_name(ZFS_PROP_REFQUOTA)) != 0) {
4177                                 /*
4178                                  * Skip the special case of, for example,
4179                                  * "refquota", errors on intermediate
4180                                  * snapshots leading up to a final one.
4181                                  * That's why we have all of the checks above.
4182                                  *
4183                                  * See zfs_ioctl.c's extract_delay_props() for
4184                                  * a list of props which can fail on
4185                                  * intermediate snapshots, but shouldn't
4186                                  * affect the overall receive.
4187                                  */
4188                                 (void) snprintf(tbuf, sizeof (tbuf),
4189                                     dgettext(TEXT_DOMAIN,
4190                                     "cannot receive %s property on %s"),
4191                                     nvpair_name(prop_err), name);
4192                                 zfs_setprop_error(hdl, prop, intval, tbuf);
4193                         }
4194                 }
4195         }
4196
4197         if (err == 0 && snapprops_nvlist) {
4198                 zfs_cmd_t zc = {"\0"};
4199
4200                 (void) strcpy(zc.zc_name, destsnap);
4201                 zc.zc_cookie = B_TRUE; /* received */
4202                 if (zcmd_write_src_nvlist(hdl, &zc, snapprops_nvlist) == 0) {
4203                         (void) zfs_ioctl(hdl, ZFS_IOC_SET_PROP, &zc);
4204                         zcmd_free_nvlists(&zc);
4205                 }
4206         }
4207
4208         if (err && (ioctl_errno == ENOENT || ioctl_errno == EEXIST)) {
4209                 /*
4210                  * It may be that this snapshot already exists,
4211                  * in which case we want to consume & ignore it
4212                  * rather than failing.
4213                  */
4214                 avl_tree_t *local_avl;
4215                 nvlist_t *local_nv, *fs;
4216                 cp = strchr(destsnap, '@');
4217
4218                 /*
4219                  * XXX Do this faster by just iterating over snaps in
4220                  * this fs.  Also if zc_value does not exist, we will
4221                  * get a strange "does not exist" error message.
4222                  */
4223                 *cp = '\0';
4224                 if (gather_nvlist(hdl, destsnap, NULL, NULL, B_FALSE, B_TRUE,
4225                     B_FALSE, B_FALSE, &local_nv, &local_avl) == 0) {
4226                         *cp = '@';
4227                         fs = fsavl_find(local_avl, drrb->drr_toguid, NULL);
4228                         fsavl_destroy(local_avl);
4229                         nvlist_free(local_nv);
4230
4231                         if (fs != NULL) {
4232                                 if (flags->verbose) {
4233                                         (void) printf("snap %s already exists; "
4234                                             "ignoring\n", destsnap);
4235                                 }
4236                                 err = ioctl_err = recv_skip(hdl, infd,
4237                                     flags->byteswap);
4238                         }
4239                 }
4240                 *cp = '@';
4241         }
4242
4243         if (ioctl_err != 0) {
4244                 switch (ioctl_errno) {
4245                 case ENODEV:
4246                         cp = strchr(destsnap, '@');
4247                         *cp = '\0';
4248                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4249                             "most recent snapshot of %s does not\n"
4250                             "match incremental source"), destsnap);
4251                         (void) zfs_error(hdl, EZFS_BADRESTORE, errbuf);
4252                         *cp = '@';
4253                         break;
4254                 case ETXTBSY:
4255                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4256                             "destination %s has been modified\n"
4257                             "since most recent snapshot"), name);
4258                         (void) zfs_error(hdl, EZFS_BADRESTORE, errbuf);
4259                         break;
4260                 case EACCES:
4261                         if (raw && stream_wantsnewfs) {
4262                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4263                                     "failed to create encryption key"));
4264                         } else if (raw && !stream_wantsnewfs) {
4265                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4266                                     "encryption key does not match "
4267                                     "existing key"));
4268                         } else {
4269                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4270                                     "inherited key must be loaded"));
4271                         }
4272                         (void) zfs_error(hdl, EZFS_CRYPTOFAILED, errbuf);
4273                         break;
4274                 case EEXIST:
4275                         cp = strchr(destsnap, '@');
4276                         if (newfs) {
4277                                 /* it's the containing fs that exists */
4278                                 *cp = '\0';
4279                         }
4280                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4281                             "destination already exists"));
4282                         (void) zfs_error_fmt(hdl, EZFS_EXISTS,
4283                             dgettext(TEXT_DOMAIN, "cannot restore to %s"),
4284                             destsnap);
4285                         *cp = '@';
4286                         break;
4287                 case EINVAL:
4288                         if (flags->resumable)
4289                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4290                                     "kernel modules must be upgraded to "
4291                                     "receive this stream."));
4292                         if (embedded && !raw)
4293                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4294                                     "incompatible embedded data stream "
4295                                     "feature with encrypted receive."));
4296                         (void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
4297                         break;
4298                 case ECKSUM:
4299                         recv_ecksum_set_aux(hdl, destsnap, flags->resumable);
4300                         (void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
4301                         break;
4302                 case ENOTSUP:
4303                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4304                             "pool must be upgraded to receive this stream."));
4305                         (void) zfs_error(hdl, EZFS_BADVERSION, errbuf);
4306                         break;
4307                 case EDQUOT:
4308                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4309                             "destination %s space quota exceeded."), name);
4310                         (void) zfs_error(hdl, EZFS_NOSPC, errbuf);
4311                         break;
4312                 case EBUSY:
4313                         if (hastoken) {
4314                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4315                                     "destination %s contains "
4316                                     "partially-complete state from "
4317                                     "\"zfs receive -s\"."), name);
4318                                 (void) zfs_error(hdl, EZFS_BUSY, errbuf);
4319                                 break;
4320                         }
4321                         /* fallthru */
4322                 default:
4323                         (void) zfs_standard_error(hdl, ioctl_errno, errbuf);
4324                 }
4325         }
4326
4327         /*
4328          * Mount the target filesystem (if created).  Also mount any
4329          * children of the target filesystem if we did a replication
4330          * receive (indicated by stream_avl being non-NULL).
4331          */
4332         cp = strchr(destsnap, '@');
4333         if (cp && (ioctl_err == 0 || !newfs)) {
4334                 zfs_handle_t *h;
4335
4336                 *cp = '\0';
4337                 h = zfs_open(hdl, destsnap,
4338                     ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
4339                 if (h != NULL) {
4340                         if (h->zfs_type == ZFS_TYPE_VOLUME) {
4341                                 *cp = '@';
4342                         } else if (newfs || stream_avl) {
4343                                 /*
4344                                  * Track the first/top of hierarchy fs,
4345                                  * for mounting and sharing later.
4346                                  */
4347                                 if (top_zfs && *top_zfs == NULL)
4348                                         *top_zfs = zfs_strdup(hdl, destsnap);
4349                         }
4350                         zfs_close(h);
4351                 }
4352                 *cp = '@';
4353         }
4354
4355         if (clp) {
4356                 if (!flags->nomount)
4357                         err |= changelist_postfix(clp);
4358                 changelist_free(clp);
4359         }
4360
4361         if (prop_errflags & ZPROP_ERR_NOCLEAR) {
4362                 (void) fprintf(stderr, dgettext(TEXT_DOMAIN, "Warning: "
4363                     "failed to clear unreceived properties on %s"), name);
4364                 (void) fprintf(stderr, "\n");
4365         }
4366         if (prop_errflags & ZPROP_ERR_NORESTORE) {
4367                 (void) fprintf(stderr, dgettext(TEXT_DOMAIN, "Warning: "
4368                     "failed to restore original properties on %s"), name);
4369                 (void) fprintf(stderr, "\n");
4370         }
4371
4372         if (err || ioctl_err) {
4373                 err = -1;
4374                 goto out;
4375         }
4376
4377         if (flags->verbose) {
4378                 char buf1[64];
4379                 char buf2[64];
4380                 uint64_t bytes = read_bytes;
4381                 time_t delta = time(NULL) - begin_time;
4382                 if (delta == 0)
4383                         delta = 1;
4384                 zfs_nicebytes(bytes, buf1, sizeof (buf1));
4385                 zfs_nicebytes(bytes/delta, buf2, sizeof (buf1));
4386
4387                 (void) printf("received %s stream in %lu seconds (%s/sec)\n",
4388                     buf1, delta, buf2);
4389         }
4390
4391         err = 0;
4392 out:
4393         if (prop_errors != NULL)
4394                 nvlist_free(prop_errors);
4395
4396         if (tmp_keylocation[0] != '\0') {
4397                 VERIFY(0 == nvlist_add_string(rcvprops,
4398                     zfs_prop_to_name(ZFS_PROP_KEYLOCATION), tmp_keylocation));
4399         }
4400
4401         if (newprops)
4402                 nvlist_free(rcvprops);
4403
4404         nvlist_free(oxprops);
4405         nvlist_free(origprops);
4406
4407         return (err);
4408 }
4409
4410 /*
4411  * Check properties we were asked to override (both -o|-x)
4412  */
4413 static boolean_t
4414 zfs_receive_checkprops(libzfs_handle_t *hdl, nvlist_t *props,
4415     const char *errbuf)
4416 {
4417         nvpair_t *nvp;
4418         zfs_prop_t prop;
4419         const char *name;
4420
4421         nvp = NULL;
4422         while ((nvp = nvlist_next_nvpair(props, nvp)) != NULL) {
4423                 name = nvpair_name(nvp);
4424                 prop = zfs_name_to_prop(name);
4425
4426                 if (prop == ZPROP_INVAL) {
4427                         if (!zfs_prop_user(name)) {
4428                                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4429                                     "invalid property '%s'"), name);
4430                                 return (B_FALSE);
4431                         }
4432                         continue;
4433                 }
4434                 /*
4435                  * "origin" is readonly but is used to receive datasets as
4436                  * clones so we don't raise an error here
4437                  */
4438                 if (prop == ZFS_PROP_ORIGIN)
4439                         continue;
4440
4441                 /* encryption params have their own verification later */
4442                 if (prop == ZFS_PROP_ENCRYPTION ||
4443                     zfs_prop_encryption_key_param(prop))
4444                         continue;
4445
4446                 /*
4447                  * cannot override readonly, set-once and other specific
4448                  * settable properties
4449                  */
4450                 if (zfs_prop_readonly(prop) || prop == ZFS_PROP_VERSION ||
4451                     prop == ZFS_PROP_VOLSIZE) {
4452                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4453                             "invalid property '%s'"), name);
4454                         return (B_FALSE);
4455                 }
4456         }
4457
4458         return (B_TRUE);
4459 }
4460
4461 static int
4462 zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap,
4463     const char *originsnap, recvflags_t *flags, int infd, const char *sendfs,
4464     nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd,
4465     uint64_t *action_handlep, const char *finalsnap, nvlist_t *cmdprops)
4466 {
4467         int err;
4468         dmu_replay_record_t drr, drr_noswap;
4469         struct drr_begin *drrb = &drr.drr_u.drr_begin;
4470         char errbuf[1024];
4471         zio_cksum_t zcksum = { { 0 } };
4472         uint64_t featureflags;
4473         int hdrtype;
4474
4475         (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
4476             "cannot receive"));
4477
4478         /* check cmdline props, raise an error if they cannot be received */
4479         if (!zfs_receive_checkprops(hdl, cmdprops, errbuf)) {
4480                 return (zfs_error(hdl, EZFS_BADPROP, errbuf));
4481         }
4482
4483         if (flags->isprefix &&
4484             !zfs_dataset_exists(hdl, tosnap, ZFS_TYPE_DATASET)) {
4485                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "specified fs "
4486                     "(%s) does not exist"), tosnap);
4487                 return (zfs_error(hdl, EZFS_NOENT, errbuf));
4488         }
4489         if (originsnap &&
4490             !zfs_dataset_exists(hdl, originsnap, ZFS_TYPE_DATASET)) {
4491                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "specified origin fs "
4492                     "(%s) does not exist"), originsnap);
4493                 return (zfs_error(hdl, EZFS_NOENT, errbuf));
4494         }
4495
4496         /* read in the BEGIN record */
4497         if (0 != (err = recv_read(hdl, infd, &drr, sizeof (drr), B_FALSE,
4498             &zcksum)))
4499                 return (err);
4500
4501         if (drr.drr_type == DRR_END || drr.drr_type == BSWAP_32(DRR_END)) {
4502                 /* It's the double end record at the end of a package */
4503                 return (ENODATA);
4504         }
4505
4506         /* the kernel needs the non-byteswapped begin record */
4507         drr_noswap = drr;
4508
4509         flags->byteswap = B_FALSE;
4510         if (drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC)) {
4511                 /*
4512                  * We computed the checksum in the wrong byteorder in
4513                  * recv_read() above; do it again correctly.
4514                  */
4515                 bzero(&zcksum, sizeof (zio_cksum_t));
4516                 fletcher_4_incremental_byteswap(&drr, sizeof (drr), &zcksum);
4517                 flags->byteswap = B_TRUE;
4518
4519                 drr.drr_type = BSWAP_32(drr.drr_type);
4520                 drr.drr_payloadlen = BSWAP_32(drr.drr_payloadlen);
4521                 drrb->drr_magic = BSWAP_64(drrb->drr_magic);
4522                 drrb->drr_versioninfo = BSWAP_64(drrb->drr_versioninfo);
4523                 drrb->drr_creation_time = BSWAP_64(drrb->drr_creation_time);
4524                 drrb->drr_type = BSWAP_32(drrb->drr_type);
4525                 drrb->drr_flags = BSWAP_32(drrb->drr_flags);
4526                 drrb->drr_toguid = BSWAP_64(drrb->drr_toguid);
4527                 drrb->drr_fromguid = BSWAP_64(drrb->drr_fromguid);
4528         }
4529
4530         if (drrb->drr_magic != DMU_BACKUP_MAGIC || drr.drr_type != DRR_BEGIN) {
4531                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
4532                     "stream (bad magic number)"));
4533                 return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
4534         }
4535
4536         featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
4537         hdrtype = DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo);
4538
4539         if (!DMU_STREAM_SUPPORTED(featureflags) ||
4540             (hdrtype != DMU_SUBSTREAM && hdrtype != DMU_COMPOUNDSTREAM)) {
4541                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4542                     "stream has unsupported feature, feature flags = %lx"),
4543                     featureflags);
4544                 return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
4545         }
4546
4547         if (strchr(drrb->drr_toname, '@') == NULL) {
4548                 zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid "
4549                     "stream (bad snapshot name)"));
4550                 return (zfs_error(hdl, EZFS_BADSTREAM, errbuf));
4551         }
4552
4553         if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == DMU_SUBSTREAM) {
4554                 char nonpackage_sendfs[ZFS_MAX_DATASET_NAME_LEN];
4555                 if (sendfs == NULL) {
4556                         /*
4557                          * We were not called from zfs_receive_package(). Get
4558                          * the fs specified by 'zfs send'.
4559                          */
4560                         char *cp;
4561                         (void) strlcpy(nonpackage_sendfs,
4562                             drr.drr_u.drr_begin.drr_toname,
4563                             sizeof (nonpackage_sendfs));
4564                         if ((cp = strchr(nonpackage_sendfs, '@')) != NULL)
4565                                 *cp = '\0';
4566                         sendfs = nonpackage_sendfs;
4567                         VERIFY(finalsnap == NULL);
4568                 }
4569                 return (zfs_receive_one(hdl, infd, tosnap, originsnap, flags,
4570                     &drr, &drr_noswap, sendfs, stream_nv, stream_avl, top_zfs,
4571                     cleanup_fd, action_handlep, finalsnap, cmdprops));
4572         } else {
4573                 assert(DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
4574                     DMU_COMPOUNDSTREAM);
4575                 return (zfs_receive_package(hdl, infd, tosnap, flags, &drr,
4576                     &zcksum, top_zfs, cleanup_fd, action_handlep, cmdprops));
4577         }
4578 }
4579
4580 /*
4581  * Restores a backup of tosnap from the file descriptor specified by infd.
4582  * Return 0 on total success, -2 if some things couldn't be
4583  * destroyed/renamed/promoted, -1 if some things couldn't be received.
4584  * (-1 will override -2, if -1 and the resumable flag was specified the
4585  * transfer can be resumed if the sending side supports it).
4586  */
4587 int
4588 zfs_receive(libzfs_handle_t *hdl, const char *tosnap, nvlist_t *props,
4589     recvflags_t *flags, int infd, avl_tree_t *stream_avl)
4590 {
4591         char *top_zfs = NULL;
4592         int err;
4593         int cleanup_fd;
4594         uint64_t action_handle = 0;
4595         struct stat sb;
4596         char *originsnap = NULL;
4597
4598         /*
4599          * The only way fstat can fail is if we do not have a valid file
4600          * descriptor.
4601          */
4602         if (fstat(infd, &sb) == -1) {
4603                 perror("fstat");
4604                 return (-2);
4605         }
4606
4607 #ifdef __linux__
4608 #ifndef F_SETPIPE_SZ
4609 #define F_SETPIPE_SZ (F_SETLEASE + 7)
4610 #endif /* F_SETPIPE_SZ */
4611
4612 #ifndef F_GETPIPE_SZ
4613 #define F_GETPIPE_SZ (F_GETLEASE + 7)
4614 #endif /* F_GETPIPE_SZ */
4615
4616         /*
4617          * It is not uncommon for gigabytes to be processed in zfs receive.
4618          * Speculatively increase the buffer size via Linux-specific fcntl()
4619          * call.
4620          */
4621         if (S_ISFIFO(sb.st_mode)) {
4622                 FILE *procf = fopen("/proc/sys/fs/pipe-max-size", "r");
4623
4624                 if (procf != NULL) {
4625                         unsigned long max_psize;
4626                         long cur_psize;
4627                         if (fscanf(procf, "%lu", &max_psize) > 0) {
4628                                 cur_psize = fcntl(infd, F_GETPIPE_SZ);
4629                                 if (cur_psize > 0 &&
4630                                     max_psize > (unsigned long) cur_psize)
4631                                         (void) fcntl(infd, F_SETPIPE_SZ,
4632                                             max_psize);
4633                         }
4634                         fclose(procf);
4635                 }
4636         }
4637 #endif /* __linux__ */
4638
4639         if (props) {
4640                 err = nvlist_lookup_string(props, "origin", &originsnap);
4641                 if (err && err != ENOENT)
4642                         return (err);
4643         }
4644
4645         cleanup_fd = open(ZFS_DEV, O_RDWR);
4646         VERIFY(cleanup_fd >= 0);
4647
4648         err = zfs_receive_impl(hdl, tosnap, originsnap, flags, infd, NULL, NULL,
4649             stream_avl, &top_zfs, cleanup_fd, &action_handle, NULL, props);
4650
4651         VERIFY(0 == close(cleanup_fd));
4652
4653         if (err == 0 && !flags->nomount && top_zfs) {
4654                 zfs_handle_t *zhp = NULL;
4655                 prop_changelist_t *clp = NULL;
4656
4657                 zhp = zfs_open(hdl, top_zfs, ZFS_TYPE_FILESYSTEM);
4658                 if (zhp != NULL) {
4659                         clp = changelist_gather(zhp, ZFS_PROP_MOUNTPOINT,
4660                             CL_GATHER_MOUNT_ALWAYS, 0);
4661                         zfs_close(zhp);
4662                         if (clp != NULL) {
4663                                 /* mount and share received datasets */
4664                                 err = changelist_postfix(clp);
4665                                 changelist_free(clp);
4666                         }
4667                 }
4668                 if (zhp == NULL || clp == NULL || err)
4669                         err = -1;
4670         }
4671         if (top_zfs)
4672                 free(top_zfs);
4673
4674         return (err);
4675 }