2 * Copyright (c) 2014 Michihiro NAKAJIMA
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 #include "archive_platform.h"
28 __FBSDID("$FreeBSD$");
48 #include "archive_endian.h"
49 #include "archive_private.h"
50 #include "archive_write_private.h"
51 #include "archive_xxhash.h"
53 #define LZ4_MAGICNUMBER 0x184d2204
56 int compression_level;
57 unsigned header_written:1;
58 unsigned version_number:1;
59 unsigned block_independence:1;
60 unsigned block_checksum:1;
61 unsigned stream_size:1;
62 unsigned stream_checksum:1;
63 unsigned preset_dictionary:1;
64 unsigned block_maximum_size:3;
65 #if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
69 size_t out_buffer_size;
70 size_t out_block_size;
72 char *in_buffer_allocated;
74 size_t in_buffer_size;
80 struct archive_write_program_data *pdata;
84 static int archive_filter_lz4_close(struct archive_write_filter *);
85 static int archive_filter_lz4_free(struct archive_write_filter *);
86 static int archive_filter_lz4_open(struct archive_write_filter *);
87 static int archive_filter_lz4_options(struct archive_write_filter *,
88 const char *, const char *);
89 static int archive_filter_lz4_write(struct archive_write_filter *,
90 const void *, size_t);
93 * Add a lz4 compression filter to this write handle.
96 archive_write_add_filter_lz4(struct archive *_a)
98 struct archive_write *a = (struct archive_write *)_a;
99 struct archive_write_filter *f = __archive_write_allocate_filter(_a);
100 struct private_data *data;
102 archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,
103 ARCHIVE_STATE_NEW, "archive_write_add_filter_lz4");
105 data = calloc(1, sizeof(*data));
107 archive_set_error(&a->archive, ENOMEM, "Out of memory");
108 return (ARCHIVE_FATAL);
112 * Setup default settings.
114 data->compression_level = 1;
115 data->version_number = 0x01;
116 data->block_independence = 1;
117 data->block_checksum = 0;
118 data->stream_size = 0;
119 data->stream_checksum = 1;
120 data->preset_dictionary = 0;
121 data->block_maximum_size = 7;
124 * Setup a filter setting.
127 f->options = &archive_filter_lz4_options;
128 f->close = &archive_filter_lz4_close;
129 f->free = &archive_filter_lz4_free;
130 f->open = &archive_filter_lz4_open;
131 f->code = ARCHIVE_FILTER_LZ4;
133 #if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
137 * We don't have lz4 library, and execute external lz4 program
140 data->pdata = __archive_write_program_allocate("lz4");
141 if (data->pdata == NULL) {
143 archive_set_error(&a->archive, ENOMEM, "Out of memory");
144 return (ARCHIVE_FATAL);
146 data->compression_level = 0;
147 archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
148 "Using external lz4 program");
149 return (ARCHIVE_WARN);
157 archive_filter_lz4_options(struct archive_write_filter *f,
158 const char *key, const char *value)
160 struct private_data *data = (struct private_data *)f->data;
162 if (strcmp(key, "compression-level") == 0) {
164 if (value == NULL || !((val = value[0] - '0') >= 1 && val <= 9) ||
166 return (ARCHIVE_WARN);
171 archive_set_error(f->archive, ARCHIVE_ERRNO_PROGRAMMER,
172 "High compression not included in this build");
173 return (ARCHIVE_FATAL);
176 data->compression_level = val;
179 if (strcmp(key, "stream-checksum") == 0) {
180 data->stream_checksum = value != NULL;
183 if (strcmp(key, "block-checksum") == 0) {
184 data->block_checksum = value != NULL;
187 if (strcmp(key, "block-size") == 0) {
188 if (value == NULL || !(value[0] >= '4' && value[0] <= '7') ||
190 return (ARCHIVE_WARN);
191 data->block_maximum_size = value[0] - '0';
194 if (strcmp(key, "block-dependence") == 0) {
195 data->block_independence = value == NULL;
199 /* Note: The "warn" return is just to inform the options
200 * supervisor that we didn't handle it. It will generate
201 * a suitable error if no one used this option. */
202 return (ARCHIVE_WARN);
205 #if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
206 /* Don't compile this if we don't have liblz4. */
208 static int drive_compressor(struct archive_write_filter *, const char *,
210 static int drive_compressor_independence(struct archive_write_filter *,
211 const char *, size_t);
212 static int drive_compressor_dependence(struct archive_write_filter *,
213 const char *, size_t);
214 static int lz4_write_stream_descriptor(struct archive_write_filter *);
215 static ssize_t lz4_write_one_block(struct archive_write_filter *, const char *,
223 archive_filter_lz4_open(struct archive_write_filter *f)
225 struct private_data *data = (struct private_data *)f->data;
227 size_t required_size;
228 static size_t bkmap[] = { 64 * 1024, 256 * 1024, 1 * 1024 * 1024,
230 size_t pre_block_size;
232 ret = __archive_write_open_filter(f->next_filter);
236 if (data->block_maximum_size < 4)
237 data->block_size = bkmap[0];
239 data->block_size = bkmap[data->block_maximum_size - 4];
241 required_size = 4 + 15 + 4 + data->block_size + 4 + 4;
242 if (data->out_buffer_size < required_size) {
243 size_t bs = required_size, bpb;
244 free(data->out_buffer);
245 if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
246 /* Buffer size should be a multiple number of
247 * the of bytes per block for performance. */
248 bpb = archive_write_get_bytes_per_block(f->archive);
256 data->out_block_size = bs;
258 data->out_buffer = malloc(bs);
259 data->out = data->out_buffer;
260 data->out_buffer_size = bs;
263 pre_block_size = (data->block_independence)? 0: 64 * 1024;
264 if (data->in_buffer_size < data->block_size + pre_block_size) {
265 free(data->in_buffer_allocated);
266 data->in_buffer_size = data->block_size;
267 data->in_buffer_allocated =
268 malloc(data->in_buffer_size + pre_block_size);
269 data->in_buffer = data->in_buffer_allocated + pre_block_size;
270 if (!data->block_independence && data->compression_level >= 3)
271 data->in_buffer = data->in_buffer_allocated;
272 data->in = data->in_buffer;
273 data->in_buffer_size = data->block_size;
276 if (data->out_buffer == NULL || data->in_buffer_allocated == NULL) {
277 archive_set_error(f->archive, ENOMEM,
278 "Can't allocate data for compression buffer");
279 return (ARCHIVE_FATAL);
282 f->write = archive_filter_lz4_write;
288 * Write data to the out stream.
290 * Returns ARCHIVE_OK if all data written, error otherwise.
293 archive_filter_lz4_write(struct archive_write_filter *f,
294 const void *buff, size_t length)
296 struct private_data *data = (struct private_data *)f->data;
297 int ret = ARCHIVE_OK;
302 /* If we haven't written a stream descriptor, we have to do it first. */
303 if (!data->header_written) {
304 ret = lz4_write_stream_descriptor(f);
305 if (ret != ARCHIVE_OK)
307 data->header_written = 1;
310 /* Update statistics */
311 data->total_in += length;
313 p = (const char *)buff;
317 /* Compress input data to output buffer */
318 size = lz4_write_one_block(f, p, remaining);
319 if (size < ARCHIVE_OK)
320 return (ARCHIVE_FATAL);
321 l = data->out - data->out_buffer;
322 if (l >= data->out_block_size) {
323 ret = __archive_write_filter(f->next_filter,
324 data->out_buffer, data->out_block_size);
325 l -= data->out_block_size;
326 memcpy(data->out_buffer,
327 data->out_buffer + data->out_block_size, l);
328 data->out = data->out_buffer + l;
329 if (ret < ARCHIVE_WARN)
340 * Finish the compression.
343 archive_filter_lz4_close(struct archive_write_filter *f)
345 struct private_data *data = (struct private_data *)f->data;
348 /* Finish compression cycle. */
349 ret = (int)lz4_write_one_block(f, NULL, 0);
352 * Write the last block and the end of the stream data.
355 /* Write End Of Stream. */
356 memset(data->out, 0, 4); data->out += 4;
357 /* Write Stream checksum if needed. */
358 if (data->stream_checksum) {
359 unsigned int checksum;
360 checksum = __archive_xxhash.XXH32_digest(
362 data->xxh32_state = NULL;
363 archive_le32enc(data->out, checksum);
366 ret = __archive_write_filter(f->next_filter,
367 data->out_buffer, data->out - data->out_buffer);
370 r1 = __archive_write_close_filter(f->next_filter);
371 return (r1 < ret ? r1 : ret);
375 archive_filter_lz4_free(struct archive_write_filter *f)
377 struct private_data *data = (struct private_data *)f->data;
379 if (data->lz4_stream != NULL) {
381 if (data->compression_level >= 3)
382 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
383 LZ4_freeStreamHC(data->lz4_stream);
385 LZ4_freeHC(data->lz4_stream);
389 #if LZ4_VERSION_MINOR >= 3
390 LZ4_freeStream(data->lz4_stream);
392 LZ4_free(data->lz4_stream);
395 free(data->out_buffer);
396 free(data->in_buffer_allocated);
397 free(data->xxh32_state);
404 lz4_write_stream_descriptor(struct archive_write_filter *f)
406 struct private_data *data = (struct private_data *)f->data;
409 sd = (uint8_t *)data->out;
410 /* Write Magic Number. */
411 archive_le32enc(&sd[0], LZ4_MAGICNUMBER);
413 sd[4] = (data->version_number << 6)
414 | (data->block_independence << 5)
415 | (data->block_checksum << 4)
416 | (data->stream_size << 3)
417 | (data->stream_checksum << 2)
418 | (data->preset_dictionary << 0);
420 sd[5] = (data->block_maximum_size << 4);
421 sd[6] = (__archive_xxhash.XXH32(&sd[4], 2, 0) >> 8) & 0xff;
423 if (data->stream_checksum)
424 data->xxh32_state = __archive_xxhash.XXH32_init(0);
426 data->xxh32_state = NULL;
431 lz4_write_one_block(struct archive_write_filter *f, const char *p,
434 struct private_data *data = (struct private_data *)f->data;
438 /* Compress remaining uncompressed data. */
439 if (data->in_buffer == data->in)
442 size_t l = data->in - data->in_buffer;
443 r = drive_compressor(f, data->in_buffer, l);
447 } else if ((data->block_independence || data->compression_level < 3) &&
448 data->in_buffer == data->in && length >= data->block_size) {
449 r = drive_compressor(f, p, data->block_size);
451 r = (ssize_t)data->block_size;
453 size_t remaining_size = data->in_buffer_size -
454 (data->in - data->in_buffer);
455 size_t l = (remaining_size > length)? length: remaining_size;
456 memcpy(data->in, p, l);
458 if (l == remaining_size) {
459 r = drive_compressor(f, data->in_buffer,
463 data->in = data->in_buffer;
473 * Utility function to push input data through compressor, writing
474 * full output blocks as necessary.
476 * Note that this handles both the regular write case (finishing ==
477 * false) and the end-of-archive case (finishing == true).
480 drive_compressor(struct archive_write_filter *f, const char *p, size_t length)
482 struct private_data *data = (struct private_data *)f->data;
484 if (data->stream_checksum)
485 __archive_xxhash.XXH32_update(data->xxh32_state,
487 if (data->block_independence)
488 return drive_compressor_independence(f, p, length);
490 return drive_compressor_dependence(f, p, length);
494 drive_compressor_independence(struct archive_write_filter *f, const char *p,
497 struct private_data *data = (struct private_data *)f->data;
498 unsigned int outsize;
501 if (data->compression_level >= 3)
502 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
503 outsize = LZ4_compress_HC(p, data->out + 4,
504 (int)length, (int)data->block_size,
505 data->compression_level);
507 outsize = LZ4_compressHC2_limitedOutput(p, data->out + 4,
508 (int)length, (int)data->block_size,
509 data->compression_level);
513 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
514 outsize = LZ4_compress_default(p, data->out + 4,
515 (int)length, (int)data->block_size);
517 outsize = LZ4_compress_limitedOutput(p, data->out + 4,
518 (int)length, (int)data->block_size);
522 /* The buffer is compressed. */
523 archive_le32enc(data->out, outsize);
526 /* The buffer is not compressed. The commpressed size was
527 * bigger than its uncompressed size. */
528 archive_le32enc(data->out, length | 0x80000000);
530 memcpy(data->out, p, length);
533 data->out += outsize;
534 if (data->block_checksum) {
535 unsigned int checksum =
536 __archive_xxhash.XXH32(data->out - outsize, outsize, 0);
537 archive_le32enc(data->out, checksum);
544 drive_compressor_dependence(struct archive_write_filter *f, const char *p,
547 struct private_data *data = (struct private_data *)f->data;
550 #define DICT_SIZE (64 * 1024)
552 if (data->compression_level >= 3) {
553 if (data->lz4_stream == NULL) {
554 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
555 data->lz4_stream = LZ4_createStreamHC();
556 LZ4_resetStreamHC(data->lz4_stream, data->compression_level);
559 LZ4_createHC(data->in_buffer_allocated);
561 if (data->lz4_stream == NULL) {
562 archive_set_error(f->archive, ENOMEM,
563 "Can't allocate data for compression"
565 return (ARCHIVE_FATAL);
569 LZ4_loadDictHC(data->lz4_stream, data->in_buffer_allocated, DICT_SIZE);
571 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
572 outsize = LZ4_compress_HC_continue(
573 data->lz4_stream, p, data->out + 4, (int)length,
574 (int)data->block_size);
576 outsize = LZ4_compressHC2_limitedOutput_continue(
577 data->lz4_stream, p, data->out + 4, (int)length,
578 (int)data->block_size, data->compression_level);
583 if (data->lz4_stream == NULL) {
584 data->lz4_stream = LZ4_createStream();
585 if (data->lz4_stream == NULL) {
586 archive_set_error(f->archive, ENOMEM,
587 "Can't allocate data for compression"
589 return (ARCHIVE_FATAL);
593 LZ4_loadDict(data->lz4_stream, data->in_buffer_allocated, DICT_SIZE);
595 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
596 outsize = LZ4_compress_fast_continue(
597 data->lz4_stream, p, data->out + 4, (int)length,
598 (int)data->block_size, 1);
600 outsize = LZ4_compress_limitedOutput_continue(
601 data->lz4_stream, p, data->out + 4, (int)length,
602 (int)data->block_size);
607 /* The buffer is compressed. */
608 archive_le32enc(data->out, outsize);
611 /* The buffer is not compressed. The commpressed size was
612 * bigger than its uncompressed size. */
613 archive_le32enc(data->out, length | 0x80000000);
615 memcpy(data->out, p, length);
618 data->out += outsize;
619 if (data->block_checksum) {
620 unsigned int checksum =
621 __archive_xxhash.XXH32(data->out - outsize, outsize, 0);
622 archive_le32enc(data->out, checksum);
626 if (length == data->block_size) {
628 if (data->compression_level >= 3) {
629 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
630 LZ4_saveDictHC(data->lz4_stream, data->in_buffer_allocated, DICT_SIZE);
632 LZ4_slideInputBufferHC(data->lz4_stream);
634 data->in_buffer = data->in_buffer_allocated + DICT_SIZE;
638 LZ4_saveDict(data->lz4_stream,
639 data->in_buffer_allocated, DICT_SIZE);
645 #else /* HAVE_LIBLZ4 */
648 archive_filter_lz4_open(struct archive_write_filter *f)
650 struct private_data *data = (struct private_data *)f->data;
651 struct archive_string as;
654 archive_string_init(&as);
655 archive_strcpy(&as, "lz4 -z -q -q");
657 /* Specify a compression level. */
658 if (data->compression_level > 0) {
659 archive_strcat(&as, " -");
660 archive_strappend_char(&as, '0' + data->compression_level);
662 /* Specify a block size. */
663 archive_strcat(&as, " -B");
664 archive_strappend_char(&as, '0' + data->block_maximum_size);
666 if (data->block_checksum)
667 archive_strcat(&as, " -BX");
668 if (data->stream_checksum == 0)
669 archive_strcat(&as, " --no-frame-crc");
670 if (data->block_independence == 0)
671 archive_strcat(&as, " -BD");
673 f->write = archive_filter_lz4_write;
675 r = __archive_write_program_open(f, data->pdata, as.s);
676 archive_string_free(&as);
681 archive_filter_lz4_write(struct archive_write_filter *f, const void *buff,
684 struct private_data *data = (struct private_data *)f->data;
686 return __archive_write_program_write(f, data->pdata, buff, length);
690 archive_filter_lz4_close(struct archive_write_filter *f)
692 struct private_data *data = (struct private_data *)f->data;
694 return __archive_write_program_close(f, data->pdata);
698 archive_filter_lz4_free(struct archive_write_filter *f)
700 struct private_data *data = (struct private_data *)f->data;
702 __archive_write_program_free(data->pdata);
707 #endif /* HAVE_LIBLZ4 */