sys/kern/subr_compressor.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2014, 2017 Mark Johnston <markj@FreeBSD.org>
   5  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are
   9  * met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in
  14  *    the documentation and/or other materials provided with the
  15  *    distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 /*
  31  * Subroutines used for writing compressed user process and kernel core dumps.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 #include "opt_gzio.h"
  36 #include "opt_zstdio.h"
  37
  38 #include <sys/param.h>
  39 #include <sys/systm.h>
  40
  41 #include <sys/compressor.h>
  42 #include <sys/endian.h>
  43 #include <sys/kernel.h>
  44 #include <sys/linker_set.h>
  45 #include <sys/malloc.h>
  46
  47 MALLOC_DEFINE(M_COMPRESS, "compressor", "kernel compression subroutines");
  48
  49 struct compressor_methods {
  50         int format;
  51         void *(* const init)(size_t, int);
  52         void (* const reset)(void *);
  53         int (* const write)(void *, void *, size_t, compressor_cb_t, void *);
  54         void (* const fini)(void *);
  55 };
  56
  57 struct compressor {
  58         const struct compressor_methods *methods;
  59         compressor_cb_t cb;
  60         void *priv;
  61         void *arg;
  62 };
  63
  64 SET_DECLARE(compressors, struct compressor_methods);
  65
  66 #ifdef GZIO
  67
  68 #include <contrib/zlib/zutil.h>
  69
  70 struct gz_stream {
  71         uint8_t         *gz_buffer;     /* output buffer */
  72         size_t          gz_bufsz;       /* output buffer size */
  73         off_t           gz_off;         /* offset into the output stream */
  74         uint32_t        gz_crc;         /* stream CRC32 */
  75         z_stream        gz_stream;      /* zlib state */
  76 };
  77
  78 static void     *gz_init(size_t maxiosize, int level);
  79 static void     gz_reset(void *stream);
  80 static int      gz_write(void *stream, void *data, size_t len, compressor_cb_t,
  81                     void *);
  82 static void     gz_fini(void *stream);
  83
  84 static void *
  85 gz_alloc(void *arg __unused, u_int n, u_int sz)
  86 {
  87
  88         /*
  89          * Memory for zlib state is allocated using M_NODUMP since it may be
  90          * used to compress a kernel dump, and we don't want zlib to attempt to
  91          * compress its own state.
  92          */
  93         return (malloc(n * sz, M_COMPRESS, M_WAITOK | M_ZERO | M_NODUMP));
  94 }
  95
  96 static void
  97 gz_free(void *arg __unused, void *ptr)
  98 {
  99
 100         free(ptr, M_COMPRESS);
 101 }
 102
 103 static void *
 104 gz_init(size_t maxiosize, int level)
 105 {
 106         struct gz_stream *s;
 107         int error;
 108
 109         s = gz_alloc(NULL, 1, roundup2(sizeof(*s), PAGE_SIZE));
 110         s->gz_buffer = gz_alloc(NULL, 1, maxiosize);
 111         s->gz_bufsz = maxiosize;
 112
 113         s->gz_stream.zalloc = gz_alloc;
 114         s->gz_stream.zfree = gz_free;
 115         s->gz_stream.opaque = NULL;
 116         s->gz_stream.next_in = Z_NULL;
 117         s->gz_stream.avail_in = 0;
 118
 119         if (level != Z_DEFAULT_COMPRESSION) {
 120                 if (level < Z_BEST_SPEED)
 121                         level = Z_BEST_SPEED;
 122                 else if (level > Z_BEST_COMPRESSION)
 123                         level = Z_BEST_COMPRESSION;
 124         }
 125
 126         error = deflateInit2(&s->gz_stream, level, Z_DEFLATED, -MAX_WBITS,
 127             DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
 128         if (error != 0)
 129                 goto fail;
 130
 131         gz_reset(s);
 132
 133         return (s);
 134
 135 fail:
 136         gz_free(NULL, s);
 137         return (NULL);
 138 }
 139
 140 static void
 141 gz_reset(void *stream)
 142 {
 143         struct gz_stream *s;
 144         uint8_t *hdr;
 145         const size_t hdrlen = 10;
 146
 147         s = stream;
 148         s->gz_off = 0;
 149         s->gz_crc = crc32(0L, Z_NULL, 0);
 150
 151         (void)deflateReset(&s->gz_stream);
 152         s->gz_stream.avail_out = s->gz_bufsz;
 153         s->gz_stream.next_out = s->gz_buffer;
 154
 155         /* Write the gzip header to the output buffer. */
 156         hdr = s->gz_buffer;
 157         memset(hdr, 0, hdrlen);
 158         hdr[0] = 0x1f;
 159         hdr[1] = 0x8b;
 160         hdr[2] = Z_DEFLATED;
 161         hdr[9] = OS_CODE;
 162         s->gz_stream.next_out += hdrlen;
 163         s->gz_stream.avail_out -= hdrlen;
 164 }
 165
 166 static int
 167 gz_write(void *stream, void *data, size_t len, compressor_cb_t cb,
 168     void *arg)
 169 {
 170         struct gz_stream *s;
 171         uint8_t trailer[8];
 172         size_t room;
 173         int error, zerror, zflag;
 174
 175         s = stream;
 176         zflag = data == NULL ? Z_FINISH : Z_NO_FLUSH;
 177
 178         if (len > 0) {
 179                 s->gz_stream.avail_in = len;
 180                 s->gz_stream.next_in = data;
 181                 s->gz_crc = crc32(s->gz_crc, data, len);
 182         }
 183
 184         error = 0;
 185         do {
 186                 zerror = deflate(&s->gz_stream, zflag);
 187                 if (zerror != Z_OK && zerror != Z_STREAM_END) {
 188                         error = EIO;
 189                         break;
 190                 }
 191
 192                 if (s->gz_stream.avail_out == 0 || zerror == Z_STREAM_END) {
 193                         /*
 194                          * Our output buffer is full or there's nothing left
 195                          * to produce, so we're flushing the buffer.
 196                          */
 197                         len = s->gz_bufsz - s->gz_stream.avail_out;
 198                         if (zerror == Z_STREAM_END) {
 199                                 /*
 200                                  * Try to pack as much of the trailer into the
 201                                  * output buffer as we can.
 202                                  */
 203                                 ((uint32_t *)trailer)[0] = htole32(s->gz_crc);
 204                                 ((uint32_t *)trailer)[1] =
 205                                     htole32(s->gz_stream.total_in);
 206                                 room = MIN(sizeof(trailer),
 207                                     s->gz_bufsz - len);
 208                                 memcpy(s->gz_buffer + len, trailer, room);
 209                                 len += room;
 210                         }
 211
 212                         error = cb(s->gz_buffer, len, s->gz_off, arg);
 213                         if (error != 0)
 214                                 break;
 215
 216                         s->gz_off += len;
 217                         s->gz_stream.next_out = s->gz_buffer;
 218                         s->gz_stream.avail_out = s->gz_bufsz;
 219
 220                         /*
 221                          * If we couldn't pack the trailer into the output
 222                          * buffer, write it out now.
 223                          */
 224                         if (zerror == Z_STREAM_END && room < sizeof(trailer))
 225                                 error = cb(trailer + room,
 226                                     sizeof(trailer) - room, s->gz_off, arg);
 227                 }
 228         } while (zerror != Z_STREAM_END &&
 229             (zflag == Z_FINISH || s->gz_stream.avail_in > 0));
 230
 231         return (error);
 232 }
 233
 234 static void
 235 gz_fini(void *stream)
 236 {
 237         struct gz_stream *s;
 238
 239         s = stream;
 240         (void)deflateEnd(&s->gz_stream);
 241         gz_free(NULL, s->gz_buffer);
 242         gz_free(NULL, s);
 243 }
 244
 245 struct compressor_methods gzip_methods = {
 246         .format = COMPRESS_GZIP,
 247         .init = gz_init,
 248         .reset = gz_reset,
 249         .write = gz_write,
 250         .fini = gz_fini,
 251 };
 252 DATA_SET(compressors, gzip_methods);
 253
 254 #endif /* GZIO */
 255
 256 #ifdef ZSTDIO
 257
 258 #define ZSTD_STATIC_LINKING_ONLY
 259 #include <contrib/zstd/lib/zstd.h>
 260
 261 struct zstdio_stream {
 262         ZSTD_CCtx       *zst_stream;
 263         ZSTD_inBuffer   zst_inbuffer;
 264         ZSTD_outBuffer  zst_outbuffer;
 265         uint8_t *       zst_buffer;     /* output buffer */
 266         size_t          zst_maxiosz;    /* Max output IO size */
 267         off_t           zst_off;        /* offset into the output stream */
 268         void *          zst_static_wkspc;
 269 };
 270
 271 static void     *zstdio_init(size_t maxiosize, int level);
 272 static void     zstdio_reset(void *stream);
 273 static int      zstdio_write(void *stream, void *data, size_t len,
 274                     compressor_cb_t, void *);
 275 static void     zstdio_fini(void *stream);
 276
 277 static void *
 278 zstdio_init(size_t maxiosize, int level)
 279 {
 280         ZSTD_CCtx *dump_compressor;
 281         struct zstdio_stream *s;
 282         void *wkspc, *owkspc, *buffer;
 283         size_t wkspc_size, buf_size, rc;
 284
 285         s = NULL;
 286         wkspc_size = ZSTD_estimateCStreamSize(level);
 287         owkspc = wkspc = malloc(wkspc_size + 8, M_COMPRESS,
 288             M_WAITOK | M_NODUMP);
 289         /* Zstd API requires 8-byte alignment. */
 290         if ((uintptr_t)wkspc % 8 != 0)
 291                 wkspc = (void *)roundup2((uintptr_t)wkspc, 8);
 292
 293         dump_compressor = ZSTD_initStaticCCtx(wkspc, wkspc_size);
 294         if (dump_compressor == NULL) {
 295                 printf("%s: workspace too small.\n", __func__);
 296                 goto out;
 297         }
 298
 299         rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_checksumFlag, 1);
 300         if (ZSTD_isError(rc)) {
 301                 printf("%s: error setting checksumFlag: %s\n", __func__,
 302                     ZSTD_getErrorName(rc));
 303                 goto out;
 304         }
 305         rc = ZSTD_CCtx_setParameter(dump_compressor, ZSTD_c_compressionLevel,
 306             level);
 307         if (ZSTD_isError(rc)) {
 308                 printf("%s: error setting compressLevel: %s\n", __func__,
 309                     ZSTD_getErrorName(rc));
 310                 goto out;
 311         }
 312
 313         buf_size = ZSTD_CStreamOutSize() * 2;
 314         buffer = malloc(buf_size, M_COMPRESS, M_WAITOK | M_NODUMP);
 315
 316         s = malloc(sizeof(*s), M_COMPRESS, M_NODUMP | M_WAITOK);
 317         s->zst_buffer = buffer;
 318         s->zst_outbuffer.dst = buffer;
 319         s->zst_outbuffer.size = buf_size;
 320         s->zst_maxiosz = maxiosize;
 321         s->zst_stream = dump_compressor;
 322         s->zst_static_wkspc = owkspc;
 323
 324         zstdio_reset(s);
 325
 326 out:
 327         if (s == NULL)
 328                 free(owkspc, M_COMPRESS);
 329         return (s);
 330 }
 331
 332 static void
 333 zstdio_reset(void *stream)
 334 {
 335         struct zstdio_stream *s;
 336         size_t res;
 337
 338         s = stream;
 339         res = ZSTD_CCtx_reset(s->zst_stream, ZSTD_reset_session_only);
 340         if (ZSTD_isError(res))
 341                 panic("%s: could not reset stream %p: %s\n", __func__, s,
 342                     ZSTD_getErrorName(res));
 343         res = ZSTD_CCtx_setPledgedSrcSize(s->zst_stream,
 344             ZSTD_CONTENTSIZE_UNKNOWN);
 345         if (ZSTD_isError(res))
 346                 panic("%s: could not set src size on %p: %s\n", __func__, s,
 347                     ZSTD_getErrorName(res));
 348
 349         s->zst_off = 0;
 350         s->zst_inbuffer.src = NULL;
 351         s->zst_inbuffer.size = 0;
 352         s->zst_inbuffer.pos = 0;
 353         s->zst_outbuffer.pos = 0;
 354 }
 355
 356 static int
 357 zst_flush_intermediate(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
 358 {
 359         size_t bytes_to_dump;
 360         int error;
 361
 362         /* Flush as many full output blocks as possible. */
 363         /* XXX: 4096 is arbitrary safe HDD block size for kernel dumps */
 364         while (s->zst_outbuffer.pos >= 4096) {
 365                 bytes_to_dump = rounddown(s->zst_outbuffer.pos, 4096);
 366
 367                 if (bytes_to_dump > s->zst_maxiosz)
 368                         bytes_to_dump = s->zst_maxiosz;
 369
 370                 error = cb(s->zst_buffer, bytes_to_dump, s->zst_off, arg);
 371                 if (error != 0)
 372                         return (error);
 373
 374                 /*
 375                  * Shift any non-full blocks up to the front of the output
 376                  * buffer.
 377                  */
 378                 s->zst_outbuffer.pos -= bytes_to_dump;
 379                 memmove(s->zst_outbuffer.dst,
 380                     (char *)s->zst_outbuffer.dst + bytes_to_dump,
 381                     s->zst_outbuffer.pos);
 382                 s->zst_off += bytes_to_dump;
 383         }
 384         return (0);
 385 }
 386
 387 static int
 388 zstdio_flush(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
 389 {
 390         size_t rc, lastpos;
 391         int error;
 392
 393         /*
 394          * Positive return indicates unflushed data remaining; need to call
 395          * endStream again after clearing out room in output buffer.
 396          */
 397         rc = 1;
 398         lastpos = s->zst_outbuffer.pos;
 399         while (rc > 0) {
 400                 rc = ZSTD_endStream(s->zst_stream, &s->zst_outbuffer);
 401                 if (ZSTD_isError(rc)) {
 402                         printf("%s: ZSTD_endStream failed (%s)\n", __func__,
 403                             ZSTD_getErrorName(rc));
 404                         return (EIO);
 405                 }
 406                 if (lastpos == s->zst_outbuffer.pos) {
 407                         printf("%s: did not make forward progress endStream %zu\n",
 408                             __func__, lastpos);
 409                         return (EIO);
 410                 }
 411
 412                 error = zst_flush_intermediate(s, cb, arg);
 413                 if (error != 0)
 414                         return (error);
 415
 416                 lastpos = s->zst_outbuffer.pos;
 417         }
 418
 419         /*
 420          * We've already done an intermediate flush, so all full blocks have
 421          * been written.  Only a partial block remains.  Padding happens in a
 422          * higher layer.
 423          */
 424         if (s->zst_outbuffer.pos != 0) {
 425                 error = cb(s->zst_buffer, s->zst_outbuffer.pos, s->zst_off,
 426                     arg);
 427                 if (error != 0)
 428                         return (error);
 429         }
 430
 431         return (0);
 432 }
 433
 434 static int
 435 zstdio_write(void *stream, void *data, size_t len, compressor_cb_t cb,
 436     void *arg)
 437 {
 438         struct zstdio_stream *s;
 439         size_t lastpos, rc;
 440         int error;
 441
 442         s = stream;
 443         if (data == NULL)
 444                 return (zstdio_flush(s, cb, arg));
 445
 446         s->zst_inbuffer.src = data;
 447         s->zst_inbuffer.size = len;
 448         s->zst_inbuffer.pos = 0;
 449         lastpos = 0;
 450
 451         while (s->zst_inbuffer.pos < s->zst_inbuffer.size) {
 452                 rc = ZSTD_compressStream(s->zst_stream, &s->zst_outbuffer,
 453                     &s->zst_inbuffer);
 454                 if (ZSTD_isError(rc)) {
 455                         printf("%s: Compress failed on %p! (%s)\n",
 456                             __func__, data, ZSTD_getErrorName(rc));
 457                         return (EIO);
 458                 }
 459
 460                 if (lastpos == s->zst_inbuffer.pos) {
 461                         /*
 462                          * XXX: May need flushStream to make forward progress
 463                          */
 464                         printf("ZSTD: did not make forward progress @pos %zu\n",
 465                             lastpos);
 466                         return (EIO);
 467                 }
 468                 lastpos = s->zst_inbuffer.pos;
 469
 470                 error = zst_flush_intermediate(s, cb, arg);
 471                 if (error != 0)
 472                         return (error);
 473         }
 474         return (0);
 475 }
 476
 477 static void
 478 zstdio_fini(void *stream)
 479 {
 480         struct zstdio_stream *s;
 481
 482         s = stream;
 483         if (s->zst_static_wkspc != NULL)
 484                 free(s->zst_static_wkspc, M_COMPRESS);
 485         else
 486                 ZSTD_freeCCtx(s->zst_stream);
 487         free(s->zst_buffer, M_COMPRESS);
 488         free(s, M_COMPRESS);
 489 }
 490
 491 static struct compressor_methods zstd_methods = {
 492         .format = COMPRESS_ZSTD,
 493         .init = zstdio_init,
 494         .reset = zstdio_reset,
 495         .write = zstdio_write,
 496         .fini = zstdio_fini,
 497 };
 498 DATA_SET(compressors, zstd_methods);
 499
 500 #endif /* ZSTDIO */
 501
 502 bool
 503 compressor_avail(int format)
 504 {
 505         struct compressor_methods **iter;
 506
 507         SET_FOREACH(iter, compressors) {
 508                 if ((*iter)->format == format)
 509                         return (true);
 510         }
 511         return (false);
 512 }
 513
 514 struct compressor *
 515 compressor_init(compressor_cb_t cb, int format, size_t maxiosize, int level,
 516     void *arg)
 517 {
 518         struct compressor_methods **iter;
 519         struct compressor *s;
 520         void *priv;
 521
 522         SET_FOREACH(iter, compressors) {
 523                 if ((*iter)->format == format)
 524                         break;
 525         }
 526         if (iter == SET_LIMIT(compressors))
 527                 return (NULL);
 528
 529         priv = (*iter)->init(maxiosize, level);
 530         if (priv == NULL)
 531                 return (NULL);
 532
 533         s = malloc(sizeof(*s), M_COMPRESS, M_WAITOK | M_ZERO);
 534         s->methods = (*iter);
 535         s->priv = priv;
 536         s->cb = cb;
 537         s->arg = arg;
 538         return (s);
 539 }
 540
 541 void
 542 compressor_reset(struct compressor *stream)
 543 {
 544
 545         stream->methods->reset(stream->priv);
 546 }
 547
 548 int
 549 compressor_write(struct compressor *stream, void *data, size_t len)
 550 {
 551
 552         return (stream->methods->write(stream->priv, data, len, stream->cb,
 553             stream->arg));
 554 }
 555
 556 int
 557 compressor_flush(struct compressor *stream)
 558 {
 559
 560         return (stream->methods->write(stream->priv, NULL, 0, stream->cb,
 561             stream->arg));
 562 }
 563
 564 void
 565 compressor_fini(struct compressor *stream)
 566 {
 567
 568         stream->methods->fini(stream->priv);
 569 }