sys/kern/subr_compressor.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2014, 2017 Mark Johnston <markj@FreeBSD.org>
   5  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are
   9  * met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in
  14  *    the documentation and/or other materials provided with the
  15  *    distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 /*
  31  * Subroutines used for writing compressed user process and kernel core dumps.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include "opt_gzio.h"
  38 #include "opt_zstdio.h"
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42
  43 #include <sys/compressor.h>
  44 #include <sys/endian.h>
  45 #include <sys/kernel.h>
  46 #include <sys/linker_set.h>
  47 #include <sys/malloc.h>
  48
  49 MALLOC_DEFINE(M_COMPRESS, "compressor", "kernel compression subroutines");
  50
  51 struct compressor_methods {
  52         int format;
  53         void *(* const init)(size_t, int);
  54         void (* const reset)(void *);
  55         int (* const write)(void *, void *, size_t, compressor_cb_t, void *);
  56         void (* const fini)(void *);
  57 };
  58
  59 struct compressor {
  60         const struct compressor_methods *methods;
  61         compressor_cb_t cb;
  62         void *priv;
  63         void *arg;
  64 };
  65
  66 SET_DECLARE(compressors, struct compressor_methods);
  67
  68 #ifdef GZIO
  69
  70 #include <contrib/zlib/zutil.h>
  71
  72 struct gz_stream {
  73         uint8_t         *gz_buffer;     /* output buffer */
  74         size_t          gz_bufsz;       /* output buffer size */
  75         off_t           gz_off;         /* offset into the output stream */
  76         uint32_t        gz_crc;         /* stream CRC32 */
  77         z_stream        gz_stream;      /* zlib state */
  78 };
  79
  80 static void     *gz_init(size_t maxiosize, int level);
  81 static void     gz_reset(void *stream);
  82 static int      gz_write(void *stream, void *data, size_t len, compressor_cb_t,
  83                     void *);
  84 static void     gz_fini(void *stream);
  85
  86 static void *
  87 gz_alloc(void *arg __unused, u_int n, u_int sz)
  88 {
  89
  90         /*
  91          * Memory for zlib state is allocated using M_NODUMP since it may be
  92          * used to compress a kernel dump, and we don't want zlib to attempt to
  93          * compress its own state.
  94          */
  95         return (malloc(n * sz, M_COMPRESS, M_WAITOK | M_ZERO | M_NODUMP));
  96 }
  97
  98 static void
  99 gz_free(void *arg __unused, void *ptr)
 100 {
 101
 102         free(ptr, M_COMPRESS);
 103 }
 104
 105 static void *
 106 gz_init(size_t maxiosize, int level)
 107 {
 108         struct gz_stream *s;
 109         int error;
 110
 111         s = gz_alloc(NULL, 1, roundup2(sizeof(*s), PAGE_SIZE));
 112         s->gz_buffer = gz_alloc(NULL, 1, maxiosize);
 113         s->gz_bufsz = maxiosize;
 114
 115         s->gz_stream.zalloc = gz_alloc;
 116         s->gz_stream.zfree = gz_free;
 117         s->gz_stream.opaque = NULL;
 118         s->gz_stream.next_in = Z_NULL;
 119         s->gz_stream.avail_in = 0;
 120
 121         if (level != Z_DEFAULT_COMPRESSION) {
 122                 if (level < Z_BEST_SPEED)
 123                         level = Z_BEST_SPEED;
 124                 else if (level > Z_BEST_COMPRESSION)
 125                         level = Z_BEST_COMPRESSION;
 126         }
 127
 128         error = deflateInit2(&s->gz_stream, level, Z_DEFLATED, -MAX_WBITS,
 129             DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
 130         if (error != 0)
 131                 goto fail;
 132
 133         gz_reset(s);
 134
 135         return (s);
 136
 137 fail:
 138         gz_free(NULL, s);
 139         return (NULL);
 140 }
 141
 142 static void
 143 gz_reset(void *stream)
 144 {
 145         struct gz_stream *s;
 146         uint8_t *hdr;
 147         const size_t hdrlen = 10;
 148
 149         s = stream;
 150         s->gz_off = 0;
 151         s->gz_crc = crc32(0L, Z_NULL, 0);
 152
 153         (void)deflateReset(&s->gz_stream);
 154         s->gz_stream.avail_out = s->gz_bufsz;
 155         s->gz_stream.next_out = s->gz_buffer;
 156
 157         /* Write the gzip header to the output buffer. */
 158         hdr = s->gz_buffer;
 159         memset(hdr, 0, hdrlen);
 160         hdr[0] = 0x1f;
 161         hdr[1] = 0x8b;
 162         hdr[2] = Z_DEFLATED;
 163         hdr[9] = OS_CODE;
 164         s->gz_stream.next_out += hdrlen;
 165         s->gz_stream.avail_out -= hdrlen;
 166 }
 167
 168 static int
 169 gz_write(void *stream, void *data, size_t len, compressor_cb_t cb,
 170     void *arg)
 171 {
 172         struct gz_stream *s;
 173         uint8_t trailer[8];
 174         size_t room;
 175         int error, zerror, zflag;
 176
 177         s = stream;
 178         zflag = data == NULL ? Z_FINISH : Z_NO_FLUSH;
 179
 180         if (len > 0) {
 181                 s->gz_stream.avail_in = len;
 182                 s->gz_stream.next_in = data;
 183                 s->gz_crc = crc32(s->gz_crc, data, len);
 184         }
 185
 186         error = 0;
 187         do {
 188                 zerror = deflate(&s->gz_stream, zflag);
 189                 if (zerror != Z_OK && zerror != Z_STREAM_END) {
 190                         error = EIO;
 191                         break;
 192                 }
 193
 194                 if (s->gz_stream.avail_out == 0 || zerror == Z_STREAM_END) {
 195                         /*
 196                          * Our output buffer is full or there's nothing left
 197                          * to produce, so we're flushing the buffer.
 198                          */
 199                         len = s->gz_bufsz - s->gz_stream.avail_out;
 200                         if (zerror == Z_STREAM_END) {
 201                                 /*
 202                                  * Try to pack as much of the trailer into the
 203                                  * output buffer as we can.
 204                                  */
 205                                 ((uint32_t *)trailer)[0] = htole32(s->gz_crc);
 206                                 ((uint32_t *)trailer)[1] =
 207                                     htole32(s->gz_stream.total_in);
 208                                 room = MIN(sizeof(trailer),
 209                                     s->gz_bufsz - len);
 210                                 memcpy(s->gz_buffer + len, trailer, room);
 211                                 len += room;
 212                         }
 213
 214                         error = cb(s->gz_buffer, len, s->gz_off, arg);
 215                         if (error != 0)
 216                                 break;
 217
 218                         s->gz_off += len;
 219                         s->gz_stream.next_out = s->gz_buffer;
 220                         s->gz_stream.avail_out = s->gz_bufsz;
 221
 222                         /*
 223                          * If we couldn't pack the trailer into the output
 224                          * buffer, write it out now.
 225                          */
 226                         if (zerror == Z_STREAM_END && room < sizeof(trailer))
 227                                 error = cb(trailer + room,
 228                                     sizeof(trailer) - room, s->gz_off, arg);
 229                 }
 230         } while (zerror != Z_STREAM_END &&
 231             (zflag == Z_FINISH || s->gz_stream.avail_in > 0));
 232
 233         return (error);
 234 }
 235
 236 static void
 237 gz_fini(void *stream)
 238 {
 239         struct gz_stream *s;
 240
 241         s = stream;
 242         (void)deflateEnd(&s->gz_stream);
 243         gz_free(NULL, s->gz_buffer);
 244         gz_free(NULL, s);
 245 }
 246
 247 struct compressor_methods gzip_methods = {
 248         .format = COMPRESS_GZIP,
 249         .init = gz_init,
 250         .reset = gz_reset,
 251         .write = gz_write,
 252         .fini = gz_fini,
 253 };
 254 DATA_SET(compressors, gzip_methods);
 255
 256 #endif /* GZIO */
 257
 258 #ifdef ZSTDIO
 259
 260 #define ZSTD_STATIC_LINKING_ONLY
 261 #include <contrib/zstd/lib/zstd.h>
 262
 263 struct zstdio_stream {
 264         ZSTD_CCtx       *zst_stream;
 265         ZSTD_inBuffer   zst_inbuffer;
 266         ZSTD_outBuffer  zst_outbuffer;
 267         uint8_t *       zst_buffer;     /* output buffer */
 268         size_t          zst_maxiosz;    /* Max output IO size */
 269         off_t           zst_off;        /* offset into the output stream */
 270         void *          zst_static_wkspc;
 271 };
 272
 273 static void     *zstdio_init(size_t maxiosize, int level);
 274 static void     zstdio_reset(void *stream);
 275 static int      zstdio_write(void *stream, void *data, size_t len,
 276                     compressor_cb_t, void *);
 277 static void     zstdio_fini(void *stream);
 278
 279 static void *
 280 zstdio_init(size_t maxiosize, int level)
 281 {
 282         ZSTD_CCtx *dump_compressor;
 283         struct zstdio_stream *s;
 284         void *wkspc, *owkspc, *buffer;
 285         size_t wkspc_size, buf_size, rc;
 286
 287         s = NULL;
 288         wkspc_size = ZSTD_estimateCStreamSize(level);
 289         owkspc = wkspc = malloc(wkspc_size + 8, M_COMPRESS,
 290             M_WAITOK | M_NODUMP);
 291         /* Zstd API requires 8-byte alignment. */
 292         if ((uintptr_t)wkspc % 8 != 0)
 293                 wkspc = (void *)roundup2((uintptr_t)wkspc, 8);
 294
 295         dump_compressor = ZSTD_initStaticCCtx(wkspc, wkspc_size);
 296         if (dump_compressor == NULL) {
 297                 printf("%s: workspace too small.\n", __func__);
 298                 goto out;
 299         }
 300
 301         rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_checksumFlag, 1);
 302         if (ZSTD_isError(rc)) {
 303                 printf("%s: error setting checksumFlag: %s\n", __func__,
 304                     ZSTD_getErrorName(rc));
 305                 goto out;
 306         }
 307         rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_compressionLevel,
 308             level);
 309         if (ZSTD_isError(rc)) {
 310                 printf("%s: error setting compressLevel: %s\n", __func__,
 311                     ZSTD_getErrorName(rc));
 312                 goto out;
 313         }
 314
 315         buf_size = ZSTD_CStreamOutSize() * 2;
 316         buffer = malloc(buf_size, M_COMPRESS, M_WAITOK | M_NODUMP);
 317
 318         s = malloc(sizeof(*s), M_COMPRESS, M_NODUMP | M_WAITOK);
 319         s->zst_buffer = buffer;
 320         s->zst_outbuffer.dst = buffer;
 321         s->zst_outbuffer.size = buf_size;
 322         s->zst_maxiosz = maxiosize;
 323         s->zst_stream = dump_compressor;
 324         s->zst_static_wkspc = owkspc;
 325
 326         zstdio_reset(s);
 327
 328 out:
 329         if (s == NULL)
 330                 free(owkspc, M_COMPRESS);
 331         return (s);
 332 }
 333
 334 static void
 335 zstdio_reset(void *stream)
 336 {
 337         struct zstdio_stream *s;
 338         size_t res;
 339
 340         s = stream;
 341         res = ZSTD_resetCStream(s->zst_stream, 0);
 342         if (ZSTD_isError(res))
 343                 panic("%s: could not reset stream %p: %s\n", __func__, s,
 344                     ZSTD_getErrorName(res));
 345
 346         s->zst_off = 0;
 347         s->zst_inbuffer.src = NULL;
 348         s->zst_inbuffer.size = 0;
 349         s->zst_inbuffer.pos = 0;
 350         s->zst_outbuffer.pos = 0;
 351 }
 352
 353 static int
 354 zst_flush_intermediate(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
 355 {
 356         size_t bytes_to_dump;
 357         int error;
 358
 359         /* Flush as many full output blocks as possible. */
 360         /* XXX: 4096 is arbitrary safe HDD block size for kernel dumps */
 361         while (s->zst_outbuffer.pos >= 4096) {
 362                 bytes_to_dump = rounddown(s->zst_outbuffer.pos, 4096);
 363
 364                 if (bytes_to_dump > s->zst_maxiosz)
 365                         bytes_to_dump = s->zst_maxiosz;
 366
 367                 error = cb(s->zst_buffer, bytes_to_dump, s->zst_off, arg);
 368                 if (error != 0)
 369                         return (error);
 370
 371                 /*
 372                  * Shift any non-full blocks up to the front of the output
 373                  * buffer.
 374                  */
 375                 s->zst_outbuffer.pos -= bytes_to_dump;
 376                 memmove(s->zst_outbuffer.dst,
 377                     (char *)s->zst_outbuffer.dst + bytes_to_dump,
 378                     s->zst_outbuffer.pos);
 379                 s->zst_off += bytes_to_dump;
 380         }
 381         return (0);
 382 }
 383
 384 static int
 385 zstdio_flush(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
 386 {
 387         size_t rc, lastpos;
 388         int error;
 389
 390         /*
 391          * Positive return indicates unflushed data remaining; need to call
 392          * endStream again after clearing out room in output buffer.
 393          */
 394         rc = 1;
 395         lastpos = s->zst_outbuffer.pos;
 396         while (rc > 0) {
 397                 rc = ZSTD_endStream(s->zst_stream, &s->zst_outbuffer);
 398                 if (ZSTD_isError(rc)) {
 399                         printf("%s: ZSTD_endStream failed (%s)\n", __func__,
 400                             ZSTD_getErrorName(rc));
 401                         return (EIO);
 402                 }
 403                 if (lastpos == s->zst_outbuffer.pos) {
 404                         printf("%s: did not make forward progress endStream %zu\n",
 405                             __func__, lastpos);
 406                         return (EIO);
 407                 }
 408
 409                 error = zst_flush_intermediate(s, cb, arg);
 410                 if (error != 0)
 411                         return (error);
 412
 413                 lastpos = s->zst_outbuffer.pos;
 414         }
 415
 416         /*
 417          * We've already done an intermediate flush, so all full blocks have
 418          * been written.  Only a partial block remains.  Padding happens in a
 419          * higher layer.
 420          */
 421         if (s->zst_outbuffer.pos != 0) {
 422                 error = cb(s->zst_buffer, s->zst_outbuffer.pos, s->zst_off,
 423                     arg);
 424                 if (error != 0)
 425                         return (error);
 426         }
 427
 428         return (0);
 429 }
 430
 431 static int
 432 zstdio_write(void *stream, void *data, size_t len, compressor_cb_t cb,
 433     void *arg)
 434 {
 435         struct zstdio_stream *s;
 436         size_t lastpos, rc;
 437         int error;
 438
 439         s = stream;
 440         if (data == NULL)
 441                 return (zstdio_flush(s, cb, arg));
 442
 443         s->zst_inbuffer.src = data;
 444         s->zst_inbuffer.size = len;
 445         s->zst_inbuffer.pos = 0;
 446         lastpos = 0;
 447
 448         while (s->zst_inbuffer.pos < s->zst_inbuffer.size) {
 449                 rc = ZSTD_compressStream(s->zst_stream, &s->zst_outbuffer,
 450                     &s->zst_inbuffer);
 451                 if (ZSTD_isError(rc)) {
 452                         printf("%s: Compress failed on %p! (%s)\n",
 453                             __func__, data, ZSTD_getErrorName(rc));
 454                         return (EIO);
 455                 }
 456
 457                 if (lastpos == s->zst_inbuffer.pos) {
 458                         /*
 459                          * XXX: May need flushStream to make forward progress
 460                          */
 461                         printf("ZSTD: did not make forward progress @pos %zu\n",
 462                             lastpos);
 463                         return (EIO);
 464                 }
 465                 lastpos = s->zst_inbuffer.pos;
 466
 467                 error = zst_flush_intermediate(s, cb, arg);
 468                 if (error != 0)
 469                         return (error);
 470         }
 471         return (0);
 472 }
 473
 474 static void
 475 zstdio_fini(void *stream)
 476 {
 477         struct zstdio_stream *s;
 478
 479         s = stream;
 480         if (s->zst_static_wkspc != NULL)
 481                 free(s->zst_static_wkspc, M_COMPRESS);
 482         else
 483                 ZSTD_freeCCtx(s->zst_stream);
 484         free(s->zst_buffer, M_COMPRESS);
 485         free(s, M_COMPRESS);
 486 }
 487
 488 static struct compressor_methods zstd_methods = {
 489         .format = COMPRESS_ZSTD,
 490         .init = zstdio_init,
 491         .reset = zstdio_reset,
 492         .write = zstdio_write,
 493         .fini = zstdio_fini,
 494 };
 495 DATA_SET(compressors, zstd_methods);
 496
 497 #endif /* ZSTDIO */
 498
 499 bool
 500 compressor_avail(int format)
 501 {
 502         struct compressor_methods **iter;
 503
 504         SET_FOREACH(iter, compressors) {
 505                 if ((*iter)->format == format)
 506                         return (true);
 507         }
 508         return (false);
 509 }
 510
 511 struct compressor *
 512 compressor_init(compressor_cb_t cb, int format, size_t maxiosize, int level,
 513     void *arg)
 514 {
 515         struct compressor_methods **iter;
 516         struct compressor *s;
 517         void *priv;
 518
 519         SET_FOREACH(iter, compressors) {
 520                 if ((*iter)->format == format)
 521                         break;
 522         }
 523         if (iter == SET_LIMIT(compressors))
 524                 return (NULL);
 525
 526         priv = (*iter)->init(maxiosize, level);
 527         if (priv == NULL)
 528                 return (NULL);
 529
 530         s = malloc(sizeof(*s), M_COMPRESS, M_WAITOK | M_ZERO);
 531         s->methods = (*iter);
 532         s->priv = priv;
 533         s->cb = cb;
 534         s->arg = arg;
 535         return (s);
 536 }
 537
 538 void
 539 compressor_reset(struct compressor *stream)
 540 {
 541
 542         stream->methods->reset(stream->priv);
 543 }
 544
 545 int
 546 compressor_write(struct compressor *stream, void *data, size_t len)
 547 {
 548
 549         return (stream->methods->write(stream->priv, data, len, stream->cb,
 550             stream->arg));
 551 }
 552
 553 int
 554 compressor_flush(struct compressor *stream)
 555 {
 556
 557         return (stream->methods->write(stream->priv, NULL, 0, stream->cb,
 558             stream->arg));
 559 }
 560
 561 void
 562 compressor_fini(struct compressor *stream)
 563 {
 564
 565         stream->methods->fini(stream->priv);
 566 }