contrib/libarchive/libarchive/archive_write_add_filter_xz.c

   1 /*-
   2  * Copyright (c) 2003-2010 Tim Kientzle
   3  * Copyright (c) 2009-2012 Michihiro NAKAJIMA
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include "archive_platform.h"
  28
  29 __FBSDID("$FreeBSD: head/lib/libarchive/archive_write_set_compression_xz.c 201108 2009-12-28 03:28:21Z kientzle $");
  30
  31 #ifdef HAVE_ERRNO_H
  32 #include <errno.h>
  33 #endif
  34 #ifdef HAVE_STDLIB_H
  35 #include <stdlib.h>
  36 #endif
  37 #ifdef HAVE_STRING_H
  38 #include <string.h>
  39 #endif
  40 #include <time.h>
  41 #ifdef HAVE_LZMA_H
  42 #include <lzma.h>
  43 #endif
  44
  45 #include "archive.h"
  46 #include "archive_endian.h"
  47 #include "archive_private.h"
  48 #include "archive_write_private.h"
  49
  50 #if ARCHIVE_VERSION_NUMBER < 4000000
  51 int
  52 archive_write_set_compression_lzip(struct archive *a)
  53 {
  54         __archive_write_filters_free(a);
  55         return (archive_write_add_filter_lzip(a));
  56 }
  57
  58 int
  59 archive_write_set_compression_lzma(struct archive *a)
  60 {
  61         __archive_write_filters_free(a);
  62         return (archive_write_add_filter_lzma(a));
  63 }
  64
  65 int
  66 archive_write_set_compression_xz(struct archive *a)
  67 {
  68         __archive_write_filters_free(a);
  69         return (archive_write_add_filter_xz(a));
  70 }
  71
  72 #endif
  73
  74 #ifndef HAVE_LZMA_H
  75 int
  76 archive_write_add_filter_xz(struct archive *a)
  77 {
  78         archive_set_error(a, ARCHIVE_ERRNO_MISC,
  79             "xz compression not supported on this platform");
  80         return (ARCHIVE_FATAL);
  81 }
  82
  83 int
  84 archive_write_add_filter_lzma(struct archive *a)
  85 {
  86         archive_set_error(a, ARCHIVE_ERRNO_MISC,
  87             "lzma compression not supported on this platform");
  88         return (ARCHIVE_FATAL);
  89 }
  90
  91 int
  92 archive_write_add_filter_lzip(struct archive *a)
  93 {
  94         archive_set_error(a, ARCHIVE_ERRNO_MISC,
  95             "lzma compression not supported on this platform");
  96         return (ARCHIVE_FATAL);
  97 }
  98 #else
  99 /* Don't compile this if we don't have liblzma. */
 100
 101 struct private_data {
 102         int              compression_level;
 103         uint32_t         threads;
 104         lzma_stream      stream;
 105         lzma_filter      lzmafilters[2];
 106         lzma_options_lzma lzma_opt;
 107         int64_t          total_in;
 108         unsigned char   *compressed;
 109         size_t           compressed_buffer_size;
 110         int64_t          total_out;
 111         /* the CRC32 value of uncompressed data for lzip */
 112         uint32_t         crc32;
 113 };
 114
 115 static int      archive_compressor_xz_options(struct archive_write_filter *,
 116                     const char *, const char *);
 117 static int      archive_compressor_xz_open(struct archive_write_filter *);
 118 static int      archive_compressor_xz_write(struct archive_write_filter *,
 119                     const void *, size_t);
 120 static int      archive_compressor_xz_close(struct archive_write_filter *);
 121 static int      archive_compressor_xz_free(struct archive_write_filter *);
 122 static int      drive_compressor(struct archive_write_filter *,
 123                     struct private_data *, int finishing);
 124
 125 struct option_value {
 126         uint32_t dict_size;
 127         uint32_t nice_len;
 128         lzma_match_finder mf;
 129 };
 130 static const struct option_value option_values[] = {
 131         { 1 << 16, 32, LZMA_MF_HC3},
 132         { 1 << 20, 32, LZMA_MF_HC3},
 133         { 3 << 19, 32, LZMA_MF_HC4},
 134         { 1 << 21, 32, LZMA_MF_BT4},
 135         { 3 << 20, 32, LZMA_MF_BT4},
 136         { 1 << 22, 32, LZMA_MF_BT4},
 137         { 1 << 23, 64, LZMA_MF_BT4},
 138         { 1 << 24, 64, LZMA_MF_BT4},
 139         { 3 << 23, 64, LZMA_MF_BT4},
 140         { 1 << 25, 64, LZMA_MF_BT4}
 141 };
 142
 143 static int
 144 common_setup(struct archive_write_filter *f)
 145 {
 146         struct private_data *data;
 147         struct archive_write *a = (struct archive_write *)f->archive;
 148         data = calloc(1, sizeof(*data));
 149         if (data == NULL) {
 150                 archive_set_error(&a->archive, ENOMEM, "Out of memory");
 151                 return (ARCHIVE_FATAL);
 152         }
 153         f->data = data;
 154         data->compression_level = LZMA_PRESET_DEFAULT;
 155         data->threads = 1;
 156         f->open = &archive_compressor_xz_open;
 157         f->close = archive_compressor_xz_close;
 158         f->free = archive_compressor_xz_free;
 159         f->options = &archive_compressor_xz_options;
 160         return (ARCHIVE_OK);
 161 }
 162
 163 /*
 164  * Add an xz compression filter to this write handle.
 165  */
 166 int
 167 archive_write_add_filter_xz(struct archive *_a)
 168 {
 169         struct archive_write_filter *f;
 170         int r;
 171
 172         archive_check_magic(_a, ARCHIVE_WRITE_MAGIC,
 173             ARCHIVE_STATE_NEW, "archive_write_add_filter_xz");
 174         f = __archive_write_allocate_filter(_a);
 175         r = common_setup(f);
 176         if (r == ARCHIVE_OK) {
 177                 f->code = ARCHIVE_FILTER_XZ;
 178                 f->name = "xz";
 179         }
 180         return (r);
 181 }
 182
 183 /* LZMA is handled identically, we just need a different compression
 184  * code set.  (The liblzma setup looks at the code to determine
 185  * the one place that XZ and LZMA require different handling.) */
 186 int
 187 archive_write_add_filter_lzma(struct archive *_a)
 188 {
 189         struct archive_write_filter *f;
 190         int r;
 191
 192         archive_check_magic(_a, ARCHIVE_WRITE_MAGIC,
 193             ARCHIVE_STATE_NEW, "archive_write_add_filter_lzma");
 194         f = __archive_write_allocate_filter(_a);
 195         r = common_setup(f);
 196         if (r == ARCHIVE_OK) {
 197                 f->code = ARCHIVE_FILTER_LZMA;
 198                 f->name = "lzma";
 199         }
 200         return (r);
 201 }
 202
 203 int
 204 archive_write_add_filter_lzip(struct archive *_a)
 205 {
 206         struct archive_write_filter *f;
 207         int r;
 208
 209         archive_check_magic(_a, ARCHIVE_WRITE_MAGIC,
 210             ARCHIVE_STATE_NEW, "archive_write_add_filter_lzip");
 211         f = __archive_write_allocate_filter(_a);
 212         r = common_setup(f);
 213         if (r == ARCHIVE_OK) {
 214                 f->code = ARCHIVE_FILTER_LZIP;
 215                 f->name = "lzip";
 216         }
 217         return (r);
 218 }
 219
 220 static int
 221 archive_compressor_xz_init_stream(struct archive_write_filter *f,
 222     struct private_data *data)
 223 {
 224         static const lzma_stream lzma_stream_init_data = LZMA_STREAM_INIT;
 225         int ret;
 226 #ifdef HAVE_LZMA_STREAM_ENCODER_MT
 227         lzma_mt mt_options;
 228 #endif
 229
 230         data->stream = lzma_stream_init_data;
 231         data->stream.next_out = data->compressed;
 232         data->stream.avail_out = data->compressed_buffer_size;
 233         if (f->code == ARCHIVE_FILTER_XZ) {
 234 #ifdef HAVE_LZMA_STREAM_ENCODER_MT
 235                 if (data->threads != 1) {
 236                         memset(&mt_options, 0, sizeof(mt_options));
 237                         mt_options.threads = data->threads;
 238                         mt_options.timeout = 300;
 239                         mt_options.filters = data->lzmafilters;
 240                         mt_options.check = LZMA_CHECK_CRC64;
 241                         ret = lzma_stream_encoder_mt(&(data->stream),
 242                             &mt_options);
 243                 } else
 244 #endif
 245                         ret = lzma_stream_encoder(&(data->stream),
 246                             data->lzmafilters, LZMA_CHECK_CRC64);
 247         } else if (f->code == ARCHIVE_FILTER_LZMA) {
 248                 ret = lzma_alone_encoder(&(data->stream), &data->lzma_opt);
 249         } else {        /* ARCHIVE_FILTER_LZIP */
 250                 int dict_size = data->lzma_opt.dict_size;
 251                 int ds, log2dic, wedges;
 252
 253                 /* Calculate a coded dictionary size */
 254                 if (dict_size < (1 << 12) || dict_size > (1 << 27)) {
 255                         archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
 256                             "Unacceptable dictionary size for lzip: %d",
 257                             dict_size);
 258                         return (ARCHIVE_FATAL);
 259                 }
 260                 for (log2dic = 27; log2dic >= 12; log2dic--) {
 261                         if (dict_size & (1 << log2dic))
 262                                 break;
 263                 }
 264                 if (dict_size > (1 << log2dic)) {
 265                         log2dic++;
 266                         wedges =
 267                             ((1 << log2dic) - dict_size) / (1 << (log2dic - 4));
 268                 } else
 269                         wedges = 0;
 270                 ds = ((wedges << 5) & 0xe0) | (log2dic & 0x1f);
 271
 272                 data->crc32 = 0;
 273                 /* Make a header */
 274                 data->compressed[0] = 0x4C;
 275                 data->compressed[1] = 0x5A;
 276                 data->compressed[2] = 0x49;
 277                 data->compressed[3] = 0x50;
 278                 data->compressed[4] = 1;/* Version */
 279                 data->compressed[5] = (unsigned char)ds;
 280                 data->stream.next_out += 6;
 281                 data->stream.avail_out -= 6;
 282
 283                 ret = lzma_raw_encoder(&(data->stream), data->lzmafilters);
 284         }
 285         if (ret == LZMA_OK)
 286                 return (ARCHIVE_OK);
 287
 288         switch (ret) {
 289         case LZMA_MEM_ERROR:
 290                 archive_set_error(f->archive, ENOMEM,
 291                     "Internal error initializing compression library: "
 292                     "Cannot allocate memory");
 293                 break;
 294         default:
 295                 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
 296                     "Internal error initializing compression library: "
 297                     "It's a bug in liblzma");
 298                 break;
 299         }
 300         return (ARCHIVE_FATAL);
 301 }
 302
 303 /*
 304  * Setup callback.
 305  */
 306 static int
 307 archive_compressor_xz_open(struct archive_write_filter *f)
 308 {
 309         struct private_data *data = f->data;
 310         int ret;
 311
 312         if (data->compressed == NULL) {
 313                 size_t bs = 65536, bpb;
 314                 if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
 315                         /* Buffer size should be a multiple number of the of bytes
 316                          * per block for performance. */
 317                         bpb = archive_write_get_bytes_per_block(f->archive);
 318                         if (bpb > bs)
 319                                 bs = bpb;
 320                         else if (bpb != 0)
 321                                 bs -= bs % bpb;
 322                 }
 323                 data->compressed_buffer_size = bs;
 324                 data->compressed
 325                     = (unsigned char *)malloc(data->compressed_buffer_size);
 326                 if (data->compressed == NULL) {
 327                         archive_set_error(f->archive, ENOMEM,
 328                             "Can't allocate data for compression buffer");
 329                         return (ARCHIVE_FATAL);
 330                 }
 331         }
 332
 333         f->write = archive_compressor_xz_write;
 334
 335         /* Initialize compression library. */
 336         if (f->code == ARCHIVE_FILTER_LZIP) {
 337                 const struct option_value *val =
 338                     &option_values[data->compression_level];
 339
 340                 data->lzma_opt.dict_size = val->dict_size;
 341                 data->lzma_opt.preset_dict = NULL;
 342                 data->lzma_opt.preset_dict_size = 0;
 343                 data->lzma_opt.lc = LZMA_LC_DEFAULT;
 344                 data->lzma_opt.lp = LZMA_LP_DEFAULT;
 345                 data->lzma_opt.pb = LZMA_PB_DEFAULT;
 346                 data->lzma_opt.mode =
 347                     data->compression_level<= 2? LZMA_MODE_FAST:LZMA_MODE_NORMAL;
 348                 data->lzma_opt.nice_len = val->nice_len;
 349                 data->lzma_opt.mf = val->mf;
 350                 data->lzma_opt.depth = 0;
 351                 data->lzmafilters[0].id = LZMA_FILTER_LZMA1;
 352                 data->lzmafilters[0].options = &data->lzma_opt;
 353                 data->lzmafilters[1].id = LZMA_VLI_UNKNOWN;/* Terminate */
 354         } else {
 355                 if (lzma_lzma_preset(&data->lzma_opt, data->compression_level)) {
 356                         archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
 357                             "Internal error initializing compression library");
 358                 }
 359                 data->lzmafilters[0].id = LZMA_FILTER_LZMA2;
 360                 data->lzmafilters[0].options = &data->lzma_opt;
 361                 data->lzmafilters[1].id = LZMA_VLI_UNKNOWN;/* Terminate */
 362         }
 363         ret = archive_compressor_xz_init_stream(f, data);
 364         if (ret == LZMA_OK) {
 365                 f->data = data;
 366                 return (0);
 367         }
 368         return (ARCHIVE_FATAL);
 369 }
 370
 371 /*
 372  * Set write options.
 373  */
 374 static int
 375 archive_compressor_xz_options(struct archive_write_filter *f,
 376     const char *key, const char *value)
 377 {
 378         struct private_data *data = (struct private_data *)f->data;
 379
 380         if (strcmp(key, "compression-level") == 0) {
 381                 if (value == NULL || !(value[0] >= '0' && value[0] <= '9') ||
 382                     value[1] != '\0')
 383                         return (ARCHIVE_WARN);
 384                 data->compression_level = value[0] - '0';
 385                 if (data->compression_level > 9)
 386                         data->compression_level = 9;
 387                 return (ARCHIVE_OK);
 388         } else if (strcmp(key, "threads") == 0) {
 389                 char *endptr;
 390
 391                 if (value == NULL)
 392                         return (ARCHIVE_WARN);
 393                 errno = 0;
 394                 data->threads = (int)strtoul(value, &endptr, 10);
 395                 if (errno != 0 || *endptr != '\0') {
 396                         data->threads = 1;
 397                         return (ARCHIVE_WARN);
 398                 }
 399                 if (data->threads == 0) {
 400 #ifdef HAVE_LZMA_STREAM_ENCODER_MT
 401                         data->threads = lzma_cputhreads();
 402 #else
 403                         data->threads = 1;
 404 #endif
 405                 }
 406                 return (ARCHIVE_OK);
 407         }
 408
 409         /* Note: The "warn" return is just to inform the options
 410          * supervisor that we didn't handle it.  It will generate
 411          * a suitable error if no one used this option. */
 412         return (ARCHIVE_WARN);
 413 }
 414
 415 /*
 416  * Write data to the compressed stream.
 417  */
 418 static int
 419 archive_compressor_xz_write(struct archive_write_filter *f,
 420     const void *buff, size_t length)
 421 {
 422         struct private_data *data = (struct private_data *)f->data;
 423         int ret;
 424
 425         /* Update statistics */
 426         data->total_in += length;
 427         if (f->code == ARCHIVE_FILTER_LZIP)
 428                 data->crc32 = lzma_crc32(buff, length, data->crc32);
 429
 430         /* Compress input data to output buffer */
 431         data->stream.next_in = buff;
 432         data->stream.avail_in = length;
 433         if ((ret = drive_compressor(f, data, 0)) != ARCHIVE_OK)
 434                 return (ret);
 435
 436         return (ARCHIVE_OK);
 437 }
 438
 439
 440 /*
 441  * Finish the compression...
 442  */
 443 static int
 444 archive_compressor_xz_close(struct archive_write_filter *f)
 445 {
 446         struct private_data *data = (struct private_data *)f->data;
 447         int ret;
 448
 449         ret = drive_compressor(f, data, 1);
 450         if (ret == ARCHIVE_OK) {
 451                 data->total_out +=
 452                     data->compressed_buffer_size - data->stream.avail_out;
 453                 ret = __archive_write_filter(f->next_filter,
 454                     data->compressed,
 455                     data->compressed_buffer_size - data->stream.avail_out);
 456                 if (f->code == ARCHIVE_FILTER_LZIP && ret == ARCHIVE_OK) {
 457                         archive_le32enc(data->compressed, data->crc32);
 458                         archive_le64enc(data->compressed+4, data->total_in);
 459                         archive_le64enc(data->compressed+12, data->total_out + 20);
 460                         ret = __archive_write_filter(f->next_filter,
 461                             data->compressed, 20);
 462                 }
 463         }
 464         lzma_end(&(data->stream));
 465         return ret;
 466 }
 467
 468 static int
 469 archive_compressor_xz_free(struct archive_write_filter *f)
 470 {
 471         struct private_data *data = (struct private_data *)f->data;
 472         free(data->compressed);
 473         free(data);
 474         f->data = NULL;
 475         return (ARCHIVE_OK);
 476 }
 477
 478 /*
 479  * Utility function to push input data through compressor,
 480  * writing full output blocks as necessary.
 481  *
 482  * Note that this handles both the regular write case (finishing ==
 483  * false) and the end-of-archive case (finishing == true).
 484  */
 485 static int
 486 drive_compressor(struct archive_write_filter *f,
 487     struct private_data *data, int finishing)
 488 {
 489         int ret;
 490
 491         for (;;) {
 492                 if (data->stream.avail_out == 0) {
 493                         data->total_out += data->compressed_buffer_size;
 494                         ret = __archive_write_filter(f->next_filter,
 495                             data->compressed,
 496                             data->compressed_buffer_size);
 497                         if (ret != ARCHIVE_OK)
 498                                 return (ARCHIVE_FATAL);
 499                         data->stream.next_out = data->compressed;
 500                         data->stream.avail_out = data->compressed_buffer_size;
 501                 }
 502
 503                 /* If there's nothing to do, we're done. */
 504                 if (!finishing && data->stream.avail_in == 0)
 505                         return (ARCHIVE_OK);
 506
 507                 ret = lzma_code(&(data->stream),
 508                     finishing ? LZMA_FINISH : LZMA_RUN );
 509
 510                 switch (ret) {
 511                 case LZMA_OK:
 512                         /* In non-finishing case, check if compressor
 513                          * consumed everything */
 514                         if (!finishing && data->stream.avail_in == 0)
 515                                 return (ARCHIVE_OK);
 516                         /* In finishing case, this return always means
 517                          * there's more work */
 518                         break;
 519                 case LZMA_STREAM_END:
 520                         /* This return can only occur in finishing case. */
 521                         if (finishing)
 522                                 return (ARCHIVE_OK);
 523                         archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
 524                             "lzma compression data error");
 525                         return (ARCHIVE_FATAL);
 526                 case LZMA_MEMLIMIT_ERROR:
 527                         archive_set_error(f->archive, ENOMEM,
 528                             "lzma compression error: "
 529                             "%ju MiB would have been needed",
 530                             (uintmax_t)((lzma_memusage(&(data->stream))
 531                                     + 1024 * 1024 -1)
 532                                 / (1024 * 1024)));
 533                         return (ARCHIVE_FATAL);
 534                 default:
 535                         /* Any other return value indicates an error. */
 536                         archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
 537                             "lzma compression failed:"
 538                             " lzma_code() call returned status %d",
 539                             ret);
 540                         return (ARCHIVE_FATAL);
 541                 }
 542         }
 543 }
 544
 545 #endif /* HAVE_LZMA_H */