sys/kern/subr_compressor.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2014, 2017 Mark Johnston <markj@FreeBSD.org>
   5  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are
   9  * met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in
  14  *    the documentation and/or other materials provided with the
  15  *    distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 /*
  31  * Subroutines used for writing compressed user process and kernel core dumps.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include "opt_gzio.h"
  38 #include "opt_zstdio.h"
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42
  43 #include <sys/compressor.h>
  44 #include <sys/kernel.h>
  45 #include <sys/linker_set.h>
  46 #include <sys/malloc.h>
  47
  48 MALLOC_DEFINE(M_COMPRESS, "compressor", "kernel compression subroutines");
  49
  50 struct compressor_methods {
  51         int format;
  52         void *(* const init)(size_t, int);
  53         void (* const reset)(void *);
  54         int (* const write)(void *, void *, size_t, compressor_cb_t, void *);
  55         void (* const fini)(void *);
  56 };
  57
  58 struct compressor {
  59         const struct compressor_methods *methods;
  60         compressor_cb_t cb;
  61         void *priv;
  62         void *arg;
  63 };
  64
  65 SET_DECLARE(compressors, struct compressor_methods);
  66
  67 #ifdef GZIO
  68
  69 #include <sys/zutil.h>
  70
  71 struct gz_stream {
  72         uint8_t         *gz_buffer;     /* output buffer */
  73         size_t          gz_bufsz;       /* output buffer size */
  74         off_t           gz_off;         /* offset into the output stream */
  75         uint32_t        gz_crc;         /* stream CRC32 */
  76         z_stream        gz_stream;      /* zlib state */
  77 };
  78
  79 static void     *gz_init(size_t maxiosize, int level);
  80 static void     gz_reset(void *stream);
  81 static int      gz_write(void *stream, void *data, size_t len, compressor_cb_t,
  82                     void *);
  83 static void     gz_fini(void *stream);
  84
  85 static void *
  86 gz_alloc(void *arg __unused, u_int n, u_int sz)
  87 {
  88
  89         /*
  90          * Memory for zlib state is allocated using M_NODUMP since it may be
  91          * used to compress a kernel dump, and we don't want zlib to attempt to
  92          * compress its own state.
  93          */
  94         return (malloc(n * sz, M_COMPRESS, M_WAITOK | M_ZERO | M_NODUMP));
  95 }
  96
  97 static void
  98 gz_free(void *arg __unused, void *ptr)
  99 {
 100
 101         free(ptr, M_COMPRESS);
 102 }
 103
 104 static void *
 105 gz_init(size_t maxiosize, int level)
 106 {
 107         struct gz_stream *s;
 108         int error;
 109
 110         s = gz_alloc(NULL, 1, roundup2(sizeof(*s), PAGE_SIZE));
 111         s->gz_buffer = gz_alloc(NULL, 1, maxiosize);
 112         s->gz_bufsz = maxiosize;
 113
 114         s->gz_stream.zalloc = gz_alloc;
 115         s->gz_stream.zfree = gz_free;
 116         s->gz_stream.opaque = NULL;
 117         s->gz_stream.next_in = Z_NULL;
 118         s->gz_stream.avail_in = 0;
 119
 120         error = deflateInit2(&s->gz_stream, level, Z_DEFLATED, -MAX_WBITS,
 121             DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
 122         if (error != 0)
 123                 goto fail;
 124
 125         gz_reset(s);
 126
 127         return (s);
 128
 129 fail:
 130         gz_free(NULL, s);
 131         return (NULL);
 132 }
 133
 134 static void
 135 gz_reset(void *stream)
 136 {
 137         struct gz_stream *s;
 138         uint8_t *hdr;
 139         const size_t hdrlen = 10;
 140
 141         s = stream;
 142         s->gz_off = 0;
 143         s->gz_crc = ~0U;
 144
 145         (void)deflateReset(&s->gz_stream);
 146         s->gz_stream.avail_out = s->gz_bufsz;
 147         s->gz_stream.next_out = s->gz_buffer;
 148
 149         /* Write the gzip header to the output buffer. */
 150         hdr = s->gz_buffer;
 151         memset(hdr, 0, hdrlen);
 152         hdr[0] = 0x1f;
 153         hdr[1] = 0x8b;
 154         hdr[2] = Z_DEFLATED;
 155         hdr[9] = OS_CODE;
 156         s->gz_stream.next_out += hdrlen;
 157         s->gz_stream.avail_out -= hdrlen;
 158 }
 159
 160 static int
 161 gz_write(void *stream, void *data, size_t len, compressor_cb_t cb,
 162     void *arg)
 163 {
 164         struct gz_stream *s;
 165         uint8_t trailer[8];
 166         size_t room;
 167         int error, zerror, zflag;
 168
 169         s = stream;
 170         zflag = data == NULL ? Z_FINISH : Z_NO_FLUSH;
 171
 172         if (len > 0) {
 173                 s->gz_stream.avail_in = len;
 174                 s->gz_stream.next_in = data;
 175                 s->gz_crc = crc32_raw(data, len, s->gz_crc);
 176         } else
 177                 s->gz_crc ^= ~0U;
 178
 179         error = 0;
 180         do {
 181                 zerror = deflate(&s->gz_stream, zflag);
 182                 if (zerror != Z_OK && zerror != Z_STREAM_END) {
 183                         error = EIO;
 184                         break;
 185                 }
 186
 187                 if (s->gz_stream.avail_out == 0 || zerror == Z_STREAM_END) {
 188                         /*
 189                          * Our output buffer is full or there's nothing left
 190                          * to produce, so we're flushing the buffer.
 191                          */
 192                         len = s->gz_bufsz - s->gz_stream.avail_out;
 193                         if (zerror == Z_STREAM_END) {
 194                                 /*
 195                                  * Try to pack as much of the trailer into the
 196                                  * output buffer as we can.
 197                                  */
 198                                 ((uint32_t *)trailer)[0] = s->gz_crc;
 199                                 ((uint32_t *)trailer)[1] =
 200                                     s->gz_stream.total_in;
 201                                 room = MIN(sizeof(trailer),
 202                                     s->gz_bufsz - len);
 203                                 memcpy(s->gz_buffer + len, trailer, room);
 204                                 len += room;
 205                         }
 206
 207                         error = cb(s->gz_buffer, len, s->gz_off, arg);
 208                         if (error != 0)
 209                                 break;
 210
 211                         s->gz_off += len;
 212                         s->gz_stream.next_out = s->gz_buffer;
 213                         s->gz_stream.avail_out = s->gz_bufsz;
 214
 215                         /*
 216                          * If we couldn't pack the trailer into the output
 217                          * buffer, write it out now.
 218                          */
 219                         if (zerror == Z_STREAM_END && room < sizeof(trailer))
 220                                 error = cb(trailer + room,
 221                                     sizeof(trailer) - room, s->gz_off, arg);
 222                 }
 223         } while (zerror != Z_STREAM_END &&
 224             (zflag == Z_FINISH || s->gz_stream.avail_in > 0));
 225
 226         return (error);
 227 }
 228
 229 static void
 230 gz_fini(void *stream)
 231 {
 232         struct gz_stream *s;
 233
 234         s = stream;
 235         (void)deflateEnd(&s->gz_stream);
 236         gz_free(NULL, s->gz_buffer);
 237         gz_free(NULL, s);
 238 }
 239
 240 struct compressor_methods gzip_methods = {
 241         .format = COMPRESS_GZIP,
 242         .init = gz_init,
 243         .reset = gz_reset,
 244         .write = gz_write,
 245         .fini = gz_fini,
 246 };
 247 DATA_SET(compressors, gzip_methods);
 248
 249 #endif /* GZIO */
 250
 251 #ifdef ZSTDIO
 252
 253 #define ZSTD_STATIC_LINKING_ONLY
 254 #include <contrib/zstd/lib/zstd.h>
 255
 256 struct zstdio_stream {
 257         ZSTD_CCtx       *zst_stream;
 258         ZSTD_inBuffer   zst_inbuffer;
 259         ZSTD_outBuffer  zst_outbuffer;
 260         uint8_t *       zst_buffer;     /* output buffer */
 261         size_t          zst_maxiosz;    /* Max output IO size */
 262         off_t           zst_off;        /* offset into the output stream */
 263         void *          zst_static_wkspc;
 264 };
 265
 266 static void     *zstdio_init(size_t maxiosize, int level);
 267 static void     zstdio_reset(void *stream);
 268 static int      zstdio_write(void *stream, void *data, size_t len,
 269                     compressor_cb_t, void *);
 270 static void     zstdio_fini(void *stream);
 271
 272 static void *
 273 zstdio_init(size_t maxiosize, int level)
 274 {
 275         ZSTD_CCtx *dump_compressor;
 276         struct zstdio_stream *s;
 277         void *wkspc, *owkspc, *buffer;
 278         size_t wkspc_size, buf_size, rc;
 279
 280         s = NULL;
 281         wkspc_size = ZSTD_estimateCStreamSize(level);
 282         owkspc = wkspc = malloc(wkspc_size + 8, M_COMPRESS,
 283             M_WAITOK | M_NODUMP);
 284         /* Zstd API requires 8-byte alignment. */
 285         if ((uintptr_t)wkspc % 8 != 0)
 286                 wkspc = (void *)roundup2((uintptr_t)wkspc, 8);
 287
 288         dump_compressor = ZSTD_initStaticCCtx(wkspc, wkspc_size);
 289         if (dump_compressor == NULL) {
 290                 printf("%s: workspace too small.\n", __func__);
 291                 goto out;
 292         }
 293
 294         rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_checksumFlag, 1);
 295         if (ZSTD_isError(rc)) {
 296                 printf("%s: error setting checksumFlag: %s\n", __func__,
 297                     ZSTD_getErrorName(rc));
 298                 goto out;
 299         }
 300         rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_compressionLevel,
 301             level);
 302         if (ZSTD_isError(rc)) {
 303                 printf("%s: error setting compressLevel: %s\n", __func__,
 304                     ZSTD_getErrorName(rc));
 305                 goto out;
 306         }
 307
 308         buf_size = ZSTD_CStreamOutSize() * 2;
 309         buffer = malloc(buf_size, M_COMPRESS, M_WAITOK | M_NODUMP);
 310
 311         s = malloc(sizeof(*s), M_COMPRESS, M_NODUMP | M_WAITOK);
 312         s->zst_buffer = buffer;
 313         s->zst_outbuffer.dst = buffer;
 314         s->zst_outbuffer.size = buf_size;
 315         s->zst_maxiosz = maxiosize;
 316         s->zst_stream = dump_compressor;
 317         s->zst_static_wkspc = owkspc;
 318
 319         zstdio_reset(s);
 320
 321 out:
 322         if (s == NULL)
 323                 free(owkspc, M_COMPRESS);
 324         return (s);
 325 }
 326
 327 static void
 328 zstdio_reset(void *stream)
 329 {
 330         struct zstdio_stream *s;
 331         size_t res;
 332
 333         s = stream;
 334         res = ZSTD_resetCStream(s->zst_stream, 0);
 335         if (ZSTD_isError(res))
 336                 panic("%s: could not reset stream %p: %s\n", __func__, s,
 337                     ZSTD_getErrorName(res));
 338
 339         s->zst_off = 0;
 340         s->zst_inbuffer.src = NULL;
 341         s->zst_inbuffer.size = 0;
 342         s->zst_inbuffer.pos = 0;
 343         s->zst_outbuffer.pos = 0;
 344 }
 345
 346 static int
 347 zst_flush_intermediate(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
 348 {
 349         size_t bytes_to_dump;
 350         int error;
 351
 352         /* Flush as many full output blocks as possible. */
 353         /* XXX: 4096 is arbitrary safe HDD block size for kernel dumps */
 354         while (s->zst_outbuffer.pos >= 4096) {
 355                 bytes_to_dump = rounddown(s->zst_outbuffer.pos, 4096);
 356
 357                 if (bytes_to_dump > s->zst_maxiosz)
 358                         bytes_to_dump = s->zst_maxiosz;
 359
 360                 error = cb(s->zst_buffer, bytes_to_dump, s->zst_off, arg);
 361                 if (error != 0)
 362                         return (error);
 363
 364                 /*
 365                  * Shift any non-full blocks up to the front of the output
 366                  * buffer.
 367                  */
 368                 s->zst_outbuffer.pos -= bytes_to_dump;
 369                 memmove(s->zst_outbuffer.dst,
 370                     (char *)s->zst_outbuffer.dst + bytes_to_dump,
 371                     s->zst_outbuffer.pos);
 372                 s->zst_off += bytes_to_dump;
 373         }
 374         return (0);
 375 }
 376
 377 static int
 378 zstdio_flush(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
 379 {
 380         size_t rc, lastpos;
 381         int error;
 382
 383         /*
 384          * Positive return indicates unflushed data remaining; need to call
 385          * endStream again after clearing out room in output buffer.
 386          */
 387         rc = 1;
 388         lastpos = s->zst_outbuffer.pos;
 389         while (rc > 0) {
 390                 rc = ZSTD_endStream(s->zst_stream, &s->zst_outbuffer);
 391                 if (ZSTD_isError(rc)) {
 392                         printf("%s: ZSTD_endStream failed (%s)\n", __func__,
 393                             ZSTD_getErrorName(rc));
 394                         return (EIO);
 395                 }
 396                 if (lastpos == s->zst_outbuffer.pos) {
 397                         printf("%s: did not make forward progress endStream %zu\n",
 398                             __func__, lastpos);
 399                         return (EIO);
 400                 }
 401
 402                 error = zst_flush_intermediate(s, cb, arg);
 403                 if (error != 0)
 404                         return (error);
 405
 406                 lastpos = s->zst_outbuffer.pos;
 407         }
 408
 409         /*
 410          * We've already done an intermediate flush, so all full blocks have
 411          * been written.  Only a partial block remains.  Padding happens in a
 412          * higher layer.
 413          */
 414         if (s->zst_outbuffer.pos != 0) {
 415                 error = cb(s->zst_buffer, s->zst_outbuffer.pos, s->zst_off,
 416                     arg);
 417                 if (error != 0)
 418                         return (error);
 419         }
 420
 421         return (0);
 422 }
 423
 424 static int
 425 zstdio_write(void *stream, void *data, size_t len, compressor_cb_t cb,
 426     void *arg)
 427 {
 428         struct zstdio_stream *s;
 429         size_t lastpos, rc;
 430         int error;
 431
 432         s = stream;
 433         if (data == NULL)
 434                 return (zstdio_flush(s, cb, arg));
 435
 436         s->zst_inbuffer.src = data;
 437         s->zst_inbuffer.size = len;
 438         s->zst_inbuffer.pos = 0;
 439         lastpos = 0;
 440
 441         while (s->zst_inbuffer.pos < s->zst_inbuffer.size) {
 442                 rc = ZSTD_compressStream(s->zst_stream, &s->zst_outbuffer,
 443                     &s->zst_inbuffer);
 444                 if (ZSTD_isError(rc)) {
 445                         printf("%s: Compress failed on %p! (%s)\n",
 446                             __func__, data, ZSTD_getErrorName(rc));
 447                         return (EIO);
 448                 }
 449
 450                 if (lastpos == s->zst_inbuffer.pos) {
 451                         /*
 452                          * XXX: May need flushStream to make forward progress
 453                          */
 454                         printf("ZSTD: did not make forward progress @pos %zu\n",
 455                             lastpos);
 456                         return (EIO);
 457                 }
 458                 lastpos = s->zst_inbuffer.pos;
 459
 460                 error = zst_flush_intermediate(s, cb, arg);
 461                 if (error != 0)
 462                         return (error);
 463         }
 464         return (0);
 465 }
 466
 467 static void
 468 zstdio_fini(void *stream)
 469 {
 470         struct zstdio_stream *s;
 471
 472         s = stream;
 473         if (s->zst_static_wkspc != NULL)
 474                 free(s->zst_static_wkspc, M_COMPRESS);
 475         else
 476                 ZSTD_freeCCtx(s->zst_stream);
 477         free(s->zst_buffer, M_COMPRESS);
 478         free(s, M_COMPRESS);
 479 }
 480
 481 static struct compressor_methods zstd_methods = {
 482         .format = COMPRESS_ZSTD,
 483         .init = zstdio_init,
 484         .reset = zstdio_reset,
 485         .write = zstdio_write,
 486         .fini = zstdio_fini,
 487 };
 488 DATA_SET(compressors, zstd_methods);
 489
 490 #endif /* ZSTDIO */
 491
 492 bool
 493 compressor_avail(int format)
 494 {
 495         struct compressor_methods **iter;
 496
 497         SET_FOREACH(iter, compressors) {
 498                 if ((*iter)->format == format)
 499                         return (true);
 500         }
 501         return (false);
 502 }
 503
 504 struct compressor *
 505 compressor_init(compressor_cb_t cb, int format, size_t maxiosize, int level,
 506     void *arg)
 507 {
 508         struct compressor_methods **iter;
 509         struct compressor *s;
 510         void *priv;
 511
 512         SET_FOREACH(iter, compressors) {
 513                 if ((*iter)->format == format)
 514                         break;
 515         }
 516         if (iter == SET_LIMIT(compressors))
 517                 return (NULL);
 518
 519         priv = (*iter)->init(maxiosize, level);
 520         if (priv == NULL)
 521                 return (NULL);
 522
 523         s = malloc(sizeof(*s), M_COMPRESS, M_WAITOK | M_ZERO);
 524         s->methods = (*iter);
 525         s->priv = priv;
 526         s->cb = cb;
 527         s->arg = arg;
 528         return (s);
 529 }
 530
 531 void
 532 compressor_reset(struct compressor *stream)
 533 {
 534
 535         stream->methods->reset(stream->priv);
 536 }
 537
 538 int
 539 compressor_write(struct compressor *stream, void *data, size_t len)
 540 {
 541
 542         return (stream->methods->write(stream->priv, data, len, stream->cb,
 543             stream->arg));
 544 }
 545
 546 int
 547 compressor_flush(struct compressor *stream)
 548 {
 549
 550         return (stream->methods->write(stream->priv, NULL, 0, stream->cb,
 551             stream->arg));
 552 }
 553
 554 void
 555 compressor_fini(struct compressor *stream)
 556 {
 557
 558         stream->methods->fini(stream->priv);
 559 }