2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2014, 2017 Mark Johnston <markj@FreeBSD.org>
5 * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * Subroutines used for writing compressed user process and kernel core dumps.
34 #include <sys/cdefs.h>
36 #include "opt_zstdio.h"
38 #include <sys/param.h>
39 #include <sys/systm.h>
41 #include <sys/compressor.h>
42 #include <sys/endian.h>
43 #include <sys/kernel.h>
44 #include <sys/linker_set.h>
45 #include <sys/malloc.h>
47 MALLOC_DEFINE(M_COMPRESS, "compressor", "kernel compression subroutines");
49 struct compressor_methods {
51 void *(* const init)(size_t, int);
52 void (* const reset)(void *);
53 int (* const write)(void *, void *, size_t, compressor_cb_t, void *);
54 void (* const fini)(void *);
58 const struct compressor_methods *methods;
64 SET_DECLARE(compressors, struct compressor_methods);
68 #include <contrib/zlib/zutil.h>
71 uint8_t *gz_buffer; /* output buffer */
72 size_t gz_bufsz; /* output buffer size */
73 off_t gz_off; /* offset into the output stream */
74 uint32_t gz_crc; /* stream CRC32 */
75 z_stream gz_stream; /* zlib state */
78 static void *gz_init(size_t maxiosize, int level);
79 static void gz_reset(void *stream);
80 static int gz_write(void *stream, void *data, size_t len, compressor_cb_t,
82 static void gz_fini(void *stream);
85 gz_alloc(void *arg __unused, u_int n, u_int sz)
89 * Memory for zlib state is allocated using M_NODUMP since it may be
90 * used to compress a kernel dump, and we don't want zlib to attempt to
91 * compress its own state.
93 return (malloc(n * sz, M_COMPRESS, M_WAITOK | M_ZERO | M_NODUMP));
97 gz_free(void *arg __unused, void *ptr)
100 free(ptr, M_COMPRESS);
104 gz_init(size_t maxiosize, int level)
109 s = gz_alloc(NULL, 1, roundup2(sizeof(*s), PAGE_SIZE));
110 s->gz_buffer = gz_alloc(NULL, 1, maxiosize);
111 s->gz_bufsz = maxiosize;
113 s->gz_stream.zalloc = gz_alloc;
114 s->gz_stream.zfree = gz_free;
115 s->gz_stream.opaque = NULL;
116 s->gz_stream.next_in = Z_NULL;
117 s->gz_stream.avail_in = 0;
119 if (level != Z_DEFAULT_COMPRESSION) {
120 if (level < Z_BEST_SPEED)
121 level = Z_BEST_SPEED;
122 else if (level > Z_BEST_COMPRESSION)
123 level = Z_BEST_COMPRESSION;
126 error = deflateInit2(&s->gz_stream, level, Z_DEFLATED, -MAX_WBITS,
127 DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
141 gz_reset(void *stream)
145 const size_t hdrlen = 10;
149 s->gz_crc = crc32(0L, Z_NULL, 0);
151 (void)deflateReset(&s->gz_stream);
152 s->gz_stream.avail_out = s->gz_bufsz;
153 s->gz_stream.next_out = s->gz_buffer;
155 /* Write the gzip header to the output buffer. */
157 memset(hdr, 0, hdrlen);
162 s->gz_stream.next_out += hdrlen;
163 s->gz_stream.avail_out -= hdrlen;
167 gz_write(void *stream, void *data, size_t len, compressor_cb_t cb,
173 int error, zerror, zflag;
176 zflag = data == NULL ? Z_FINISH : Z_NO_FLUSH;
179 s->gz_stream.avail_in = len;
180 s->gz_stream.next_in = data;
181 s->gz_crc = crc32(s->gz_crc, data, len);
186 zerror = deflate(&s->gz_stream, zflag);
187 if (zerror != Z_OK && zerror != Z_STREAM_END) {
192 if (s->gz_stream.avail_out == 0 || zerror == Z_STREAM_END) {
194 * Our output buffer is full or there's nothing left
195 * to produce, so we're flushing the buffer.
197 len = s->gz_bufsz - s->gz_stream.avail_out;
198 if (zerror == Z_STREAM_END) {
200 * Try to pack as much of the trailer into the
201 * output buffer as we can.
203 ((uint32_t *)trailer)[0] = htole32(s->gz_crc);
204 ((uint32_t *)trailer)[1] =
205 htole32(s->gz_stream.total_in);
206 room = MIN(sizeof(trailer),
208 memcpy(s->gz_buffer + len, trailer, room);
212 error = cb(s->gz_buffer, len, s->gz_off, arg);
217 s->gz_stream.next_out = s->gz_buffer;
218 s->gz_stream.avail_out = s->gz_bufsz;
221 * If we couldn't pack the trailer into the output
222 * buffer, write it out now.
224 if (zerror == Z_STREAM_END && room < sizeof(trailer))
225 error = cb(trailer + room,
226 sizeof(trailer) - room, s->gz_off, arg);
228 } while (zerror != Z_STREAM_END &&
229 (zflag == Z_FINISH || s->gz_stream.avail_in > 0));
235 gz_fini(void *stream)
240 (void)deflateEnd(&s->gz_stream);
241 gz_free(NULL, s->gz_buffer);
245 struct compressor_methods gzip_methods = {
246 .format = COMPRESS_GZIP,
252 DATA_SET(compressors, gzip_methods);
258 #define ZSTD_STATIC_LINKING_ONLY
259 #include <contrib/zstd/lib/zstd.h>
261 struct zstdio_stream {
262 ZSTD_CCtx *zst_stream;
263 ZSTD_inBuffer zst_inbuffer;
264 ZSTD_outBuffer zst_outbuffer;
265 uint8_t * zst_buffer; /* output buffer */
266 size_t zst_maxiosz; /* Max output IO size */
267 off_t zst_off; /* offset into the output stream */
268 void * zst_static_wkspc;
271 static void *zstdio_init(size_t maxiosize, int level);
272 static void zstdio_reset(void *stream);
273 static int zstdio_write(void *stream, void *data, size_t len,
274 compressor_cb_t, void *);
275 static void zstdio_fini(void *stream);
278 zstdio_init(size_t maxiosize, int level)
280 ZSTD_CCtx *dump_compressor;
281 struct zstdio_stream *s;
282 void *wkspc, *owkspc, *buffer;
283 size_t wkspc_size, buf_size, rc;
286 wkspc_size = ZSTD_estimateCStreamSize(level);
287 owkspc = wkspc = malloc(wkspc_size + 8, M_COMPRESS,
288 M_WAITOK | M_NODUMP);
289 /* Zstd API requires 8-byte alignment. */
290 if ((uintptr_t)wkspc % 8 != 0)
291 wkspc = (void *)roundup2((uintptr_t)wkspc, 8);
293 dump_compressor = ZSTD_initStaticCCtx(wkspc, wkspc_size);
294 if (dump_compressor == NULL) {
295 printf("%s: workspace too small.\n", __func__);
299 rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_checksumFlag, 1);
300 if (ZSTD_isError(rc)) {
301 printf("%s: error setting checksumFlag: %s\n", __func__,
302 ZSTD_getErrorName(rc));
305 rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_compressionLevel,
307 if (ZSTD_isError(rc)) {
308 printf("%s: error setting compressLevel: %s\n", __func__,
309 ZSTD_getErrorName(rc));
313 buf_size = ZSTD_CStreamOutSize() * 2;
314 buffer = malloc(buf_size, M_COMPRESS, M_WAITOK | M_NODUMP);
316 s = malloc(sizeof(*s), M_COMPRESS, M_NODUMP | M_WAITOK);
317 s->zst_buffer = buffer;
318 s->zst_outbuffer.dst = buffer;
319 s->zst_outbuffer.size = buf_size;
320 s->zst_maxiosz = maxiosize;
321 s->zst_stream = dump_compressor;
322 s->zst_static_wkspc = owkspc;
328 free(owkspc, M_COMPRESS);
333 zstdio_reset(void *stream)
335 struct zstdio_stream *s;
339 res = ZSTD_resetCStream(s->zst_stream, 0);
340 if (ZSTD_isError(res))
341 panic("%s: could not reset stream %p: %s\n", __func__, s,
342 ZSTD_getErrorName(res));
345 s->zst_inbuffer.src = NULL;
346 s->zst_inbuffer.size = 0;
347 s->zst_inbuffer.pos = 0;
348 s->zst_outbuffer.pos = 0;
352 zst_flush_intermediate(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
354 size_t bytes_to_dump;
357 /* Flush as many full output blocks as possible. */
358 /* XXX: 4096 is arbitrary safe HDD block size for kernel dumps */
359 while (s->zst_outbuffer.pos >= 4096) {
360 bytes_to_dump = rounddown(s->zst_outbuffer.pos, 4096);
362 if (bytes_to_dump > s->zst_maxiosz)
363 bytes_to_dump = s->zst_maxiosz;
365 error = cb(s->zst_buffer, bytes_to_dump, s->zst_off, arg);
370 * Shift any non-full blocks up to the front of the output
373 s->zst_outbuffer.pos -= bytes_to_dump;
374 memmove(s->zst_outbuffer.dst,
375 (char *)s->zst_outbuffer.dst + bytes_to_dump,
376 s->zst_outbuffer.pos);
377 s->zst_off += bytes_to_dump;
383 zstdio_flush(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
389 * Positive return indicates unflushed data remaining; need to call
390 * endStream again after clearing out room in output buffer.
393 lastpos = s->zst_outbuffer.pos;
395 rc = ZSTD_endStream(s->zst_stream, &s->zst_outbuffer);
396 if (ZSTD_isError(rc)) {
397 printf("%s: ZSTD_endStream failed (%s)\n", __func__,
398 ZSTD_getErrorName(rc));
401 if (lastpos == s->zst_outbuffer.pos) {
402 printf("%s: did not make forward progress endStream %zu\n",
407 error = zst_flush_intermediate(s, cb, arg);
411 lastpos = s->zst_outbuffer.pos;
415 * We've already done an intermediate flush, so all full blocks have
416 * been written. Only a partial block remains. Padding happens in a
419 if (s->zst_outbuffer.pos != 0) {
420 error = cb(s->zst_buffer, s->zst_outbuffer.pos, s->zst_off,
430 zstdio_write(void *stream, void *data, size_t len, compressor_cb_t cb,
433 struct zstdio_stream *s;
439 return (zstdio_flush(s, cb, arg));
441 s->zst_inbuffer.src = data;
442 s->zst_inbuffer.size = len;
443 s->zst_inbuffer.pos = 0;
446 while (s->zst_inbuffer.pos < s->zst_inbuffer.size) {
447 rc = ZSTD_compressStream(s->zst_stream, &s->zst_outbuffer,
449 if (ZSTD_isError(rc)) {
450 printf("%s: Compress failed on %p! (%s)\n",
451 __func__, data, ZSTD_getErrorName(rc));
455 if (lastpos == s->zst_inbuffer.pos) {
457 * XXX: May need flushStream to make forward progress
459 printf("ZSTD: did not make forward progress @pos %zu\n",
463 lastpos = s->zst_inbuffer.pos;
465 error = zst_flush_intermediate(s, cb, arg);
473 zstdio_fini(void *stream)
475 struct zstdio_stream *s;
478 if (s->zst_static_wkspc != NULL)
479 free(s->zst_static_wkspc, M_COMPRESS);
481 ZSTD_freeCCtx(s->zst_stream);
482 free(s->zst_buffer, M_COMPRESS);
486 static struct compressor_methods zstd_methods = {
487 .format = COMPRESS_ZSTD,
489 .reset = zstdio_reset,
490 .write = zstdio_write,
493 DATA_SET(compressors, zstd_methods);
498 compressor_avail(int format)
500 struct compressor_methods **iter;
502 SET_FOREACH(iter, compressors) {
503 if ((*iter)->format == format)
510 compressor_init(compressor_cb_t cb, int format, size_t maxiosize, int level,
513 struct compressor_methods **iter;
514 struct compressor *s;
517 SET_FOREACH(iter, compressors) {
518 if ((*iter)->format == format)
521 if (iter == SET_LIMIT(compressors))
524 priv = (*iter)->init(maxiosize, level);
528 s = malloc(sizeof(*s), M_COMPRESS, M_WAITOK | M_ZERO);
529 s->methods = (*iter);
537 compressor_reset(struct compressor *stream)
540 stream->methods->reset(stream->priv);
544 compressor_write(struct compressor *stream, void *data, size_t len)
547 return (stream->methods->write(stream->priv, data, len, stream->cb,
552 compressor_flush(struct compressor *stream)
555 return (stream->methods->write(stream->priv, NULL, 0, stream->cb,
560 compressor_fini(struct compressor *stream)
563 stream->methods->fini(stream->priv);