usr.bin/mkuzip/mkuzip.c

   1 /*
   2  * Copyright (c) 2004-2016 Maxim Sobolev <sobomax@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/types.h>
  31 #include <sys/endian.h>
  32 #include <sys/param.h>
  33 #include <sys/sysctl.h>
  34 #include <sys/stat.h>
  35 #include <sys/uio.h>
  36 #include <netinet/in.h>
  37 #include <assert.h>
  38 #include <ctype.h>
  39 #include <err.h>
  40 #include <fcntl.h>
  41 #include <pthread.h>
  42 #include <signal.h>
  43 #include <stdint.h>
  44 #include <stdio.h>
  45 #include <stdlib.h>
  46 #include <string.h>
  47 #include <unistd.h>
  48
  49 #include "mkuzip.h"
  50 #include "mkuz_cloop.h"
  51 #include "mkuz_blockcache.h"
  52 #include "mkuz_zlib.h"
  53 #include "mkuz_lzma.h"
  54 #include "mkuz_blk.h"
  55 #include "mkuz_cfg.h"
  56 #include "mkuz_conveyor.h"
  57 #include "mkuz_format.h"
  58 #include "mkuz_fqueue.h"
  59 #include "mkuz_time.h"
  60 #include "mkuz_insize.h"
  61
  62 #define DEFAULT_CLSTSIZE        16384
  63
  64 static struct mkuz_format uzip_fmt = {
  65         .magic = CLOOP_MAGIC_ZLIB,
  66         .default_sufx = DEFAULT_SUFX_ZLIB,
  67         .f_init = &mkuz_zlib_init,
  68         .f_compress = &mkuz_zlib_compress
  69 };
  70
  71 static struct mkuz_format ulzma_fmt = {
  72         .magic = CLOOP_MAGIC_LZMA,
  73         .default_sufx = DEFAULT_SUFX_LZMA,
  74         .f_init = &mkuz_lzma_init,
  75         .f_compress = &mkuz_lzma_compress
  76 };
  77
  78 static struct mkuz_blk *readblock(int, u_int32_t);
  79 static void usage(void);
  80 static void cleanup(void);
  81
  82 static char *cleanfile = NULL;
  83
  84 static int
  85 cmp_blkno(const struct mkuz_blk *bp, void *p)
  86 {
  87         uint32_t *ap;
  88
  89         ap = (uint32_t *)p;
  90
  91         return (bp->info.blkno == *ap);
  92 }
  93
  94 int main(int argc, char **argv)
  95 {
  96         struct mkuz_cfg cfs;
  97         char *oname;
  98         uint64_t *toc;
  99         int i, io, opt, tmp;
 100         struct {
 101                 int en;
 102                 FILE *f;
 103         } summary;
 104         struct iovec iov[2];
 105         uint64_t offset, last_offset;
 106         struct cloop_header hdr;
 107         struct mkuz_conveyor *cvp;
 108         void *c_ctx;
 109         struct mkuz_blk_info *chit;
 110         size_t ncpusz, ncpu, magiclen;
 111         double st, et;
 112
 113         st = getdtime();
 114
 115         ncpusz = sizeof(size_t);
 116         if (sysctlbyname("hw.ncpu", &ncpu, &ncpusz, NULL, 0) < 0) {
 117                 ncpu = 1;
 118         } else if (ncpu > MAX_WORKERS_AUTO) {
 119                 ncpu = MAX_WORKERS_AUTO;
 120         }
 121
 122         memset(&hdr, 0, sizeof(hdr));
 123         cfs.blksz = DEFAULT_CLSTSIZE;
 124         oname = NULL;
 125         cfs.verbose = 0;
 126         cfs.no_zcomp = 0;
 127         cfs.en_dedup = 0;
 128         summary.en = 0;
 129         summary.f = stderr;
 130         cfs.handler = &uzip_fmt;
 131         cfs.nworkers = ncpu;
 132         struct mkuz_blk *iblk, *oblk;
 133
 134         while((opt = getopt(argc, argv, "o:s:vZdLSj:")) != -1) {
 135                 switch(opt) {
 136                 case 'o':
 137                         oname = optarg;
 138                         break;
 139
 140                 case 's':
 141                         tmp = atoi(optarg);
 142                         if (tmp <= 0) {
 143                                 errx(1, "invalid cluster size specified: %s",
 144                                     optarg);
 145                                 /* Not reached */
 146                         }
 147                         cfs.blksz = tmp;
 148                         break;
 149
 150                 case 'v':
 151                         cfs.verbose = 1;
 152                         break;
 153
 154                 case 'Z':
 155                         cfs.no_zcomp = 1;
 156                         break;
 157
 158                 case 'd':
 159                         cfs.en_dedup = 1;
 160                         break;
 161
 162                 case 'L':
 163                         cfs.handler = &ulzma_fmt;
 164                         break;
 165
 166                 case 'S':
 167                         summary.en = 1;
 168                         summary.f = stdout;
 169                         break;
 170
 171                 case 'j':
 172                         tmp = atoi(optarg);
 173                         if (tmp <= 0) {
 174                                 errx(1, "invalid number of compression threads"
 175                                     " specified: %s", optarg);
 176                                 /* Not reached */
 177                         }
 178                         cfs.nworkers = tmp;
 179                         break;
 180
 181                 default:
 182                         usage();
 183                         /* Not reached */
 184                 }
 185         }
 186         argc -= optind;
 187         argv += optind;
 188
 189         if (argc != 1) {
 190                 usage();
 191                 /* Not reached */
 192         }
 193
 194         magiclen = strlcpy(hdr.magic, cfs.handler->magic, sizeof(hdr.magic));
 195         assert(magiclen < sizeof(hdr.magic));
 196
 197         if (cfs.en_dedup != 0) {
 198                 hdr.magic[CLOOP_OFS_VERSN] = CLOOP_MAJVER_3;
 199                 hdr.magic[CLOOP_OFS_COMPR] =
 200                     tolower(hdr.magic[CLOOP_OFS_COMPR]);
 201         }
 202
 203         c_ctx = cfs.handler->f_init(cfs.blksz);
 204
 205         cfs.iname = argv[0];
 206         if (oname == NULL) {
 207                 asprintf(&oname, "%s%s", cfs.iname, cfs.handler->default_sufx);
 208                 if (oname == NULL) {
 209                         err(1, "can't allocate memory");
 210                         /* Not reached */
 211                 }
 212         }
 213
 214         signal(SIGHUP, exit);
 215         signal(SIGINT, exit);
 216         signal(SIGTERM, exit);
 217         signal(SIGXCPU, exit);
 218         signal(SIGXFSZ, exit);
 219         atexit(cleanup);
 220
 221         cfs.fdr = open(cfs.iname, O_RDONLY);
 222         if (cfs.fdr < 0) {
 223                 err(1, "open(%s)", cfs.iname);
 224                 /* Not reached */
 225         }
 226         cfs.isize = mkuz_get_insize(&cfs);
 227         if (cfs.isize < 0) {
 228                 errx(1, "can't determine input image size");
 229                 /* Not reached */
 230         }
 231         hdr.nblocks = cfs.isize / cfs.blksz;
 232         if ((cfs.isize % cfs.blksz) != 0) {
 233                 if (cfs.verbose != 0)
 234                         fprintf(stderr, "file size is not multiple "
 235                         "of %d, padding data\n", cfs.blksz);
 236                 hdr.nblocks++;
 237         }
 238         toc = mkuz_safe_malloc((hdr.nblocks + 1) * sizeof(*toc));
 239
 240         cfs.fdw = open(oname, (cfs.en_dedup ? O_RDWR : O_WRONLY) | O_TRUNC | O_CREAT,
 241                    S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
 242         if (cfs.fdw < 0) {
 243                 err(1, "open(%s)", oname);
 244                 /* Not reached */
 245         }
 246         cleanfile = oname;
 247
 248         /* Prepare header that we will write later when we have index ready. */
 249         iov[0].iov_base = (char *)&hdr;
 250         iov[0].iov_len = sizeof(hdr);
 251         iov[1].iov_base = (char *)toc;
 252         iov[1].iov_len = (hdr.nblocks + 1) * sizeof(*toc);
 253         offset = iov[0].iov_len + iov[1].iov_len;
 254
 255         /* Reserve space for header */
 256         lseek(cfs.fdw, offset, SEEK_SET);
 257
 258         if (cfs.verbose != 0) {
 259                 fprintf(stderr, "data size %ju bytes, number of clusters "
 260                     "%u, index length %zu bytes\n", cfs.isize,
 261                     hdr.nblocks, iov[1].iov_len);
 262         }
 263
 264         cvp = mkuz_conveyor_ctor(&cfs);
 265
 266         last_offset = 0;
 267         iblk = oblk = NULL;
 268         for(i = io = 0; iblk != MKUZ_BLK_EOF; i++) {
 269                 iblk = readblock(cfs.fdr, cfs.blksz);
 270                 mkuz_fqueue_enq(cvp->wrk_queue, iblk);
 271                 if (iblk != MKUZ_BLK_EOF &&
 272                     (i < (cfs.nworkers * ITEMS_PER_WORKER))) {
 273                         continue;
 274                 }
 275 drain:
 276                 oblk = mkuz_fqueue_deq_when(cvp->results, cmp_blkno, &io);
 277                 assert(oblk->info.blkno == (unsigned)io);
 278                 oblk->info.offset = offset;
 279                 chit = NULL;
 280                 if (cfs.en_dedup != 0 && oblk->info.len > 0) {
 281                         chit = mkuz_blkcache_regblock(cfs.fdw, oblk);
 282                         /*
 283                          * There should be at least one non-empty block
 284                          * between us and the backref'ed offset, otherwise
 285                          * we won't be able to parse that sequence correctly
 286                          * as it would be indistinguishible from another
 287                          * empty block.
 288                          */
 289                         if (chit != NULL && chit->offset == last_offset) {
 290                                 chit = NULL;
 291                         }
 292                 }
 293                 if (chit != NULL) {
 294                         toc[io] = htobe64(chit->offset);
 295                         oblk->info.len = 0;
 296                 } else {
 297                         if (oblk->info.len > 0 && write(cfs.fdw, oblk->data,
 298                             oblk->info.len) < 0) {
 299                                 err(1, "write(%s)", oname);
 300                                 /* Not reached */
 301                         }
 302                         toc[io] = htobe64(offset);
 303                         last_offset = offset;
 304                         offset += oblk->info.len;
 305                 }
 306                 if (cfs.verbose != 0) {
 307                         fprintf(stderr, "cluster #%d, in %u bytes, "
 308                             "out len=%lu offset=%lu", io, cfs.blksz,
 309                             (u_long)oblk->info.len, (u_long)be64toh(toc[io]));
 310                         if (chit != NULL) {
 311                                 fprintf(stderr, " (backref'ed to #%d)",
 312                                     chit->blkno);
 313                         }
 314                         fprintf(stderr, "\n");
 315                 }
 316                 free(oblk);
 317                 io += 1;
 318                 if (iblk == MKUZ_BLK_EOF) {
 319                         if (io < i)
 320                                 goto drain;
 321                         /* Last block, see if we need to add some padding */
 322                         if ((offset % DEV_BSIZE) == 0)
 323                                 continue;
 324                         oblk = mkuz_blk_ctor(DEV_BSIZE - (offset % DEV_BSIZE));
 325                         oblk->info.blkno = io;
 326                         oblk->info.len = oblk->alen;
 327                         if (cfs.verbose != 0) {
 328                                 fprintf(stderr, "padding data with %lu bytes "
 329                                     "so that file size is multiple of %d\n",
 330                                     (u_long)oblk->alen, DEV_BSIZE);
 331                         }
 332                         mkuz_fqueue_enq(cvp->results, oblk);
 333                         goto drain;
 334                 }
 335         }
 336
 337         close(cfs.fdr);
 338
 339         if (cfs.verbose != 0 || summary.en != 0) {
 340                 et = getdtime();
 341                 fprintf(summary.f, "compressed data to %ju bytes, saved %lld "
 342                     "bytes, %.2f%% decrease, %.2f bytes/sec.\n", offset,
 343                     (long long)(cfs.isize - offset),
 344                     100.0 * (long long)(cfs.isize - offset) /
 345                     (float)cfs.isize, (float)cfs.isize / (et - st));
 346         }
 347
 348         /* Convert to big endian */
 349         hdr.blksz = htonl(cfs.blksz);
 350         hdr.nblocks = htonl(hdr.nblocks);
 351         /* Write headers into pre-allocated space */
 352         lseek(cfs.fdw, 0, SEEK_SET);
 353         if (writev(cfs.fdw, iov, 2) < 0) {
 354                 err(1, "writev(%s)", oname);
 355                 /* Not reached */
 356         }
 357         cleanfile = NULL;
 358         close(cfs.fdw);
 359
 360         exit(0);
 361 }
 362
 363 static struct mkuz_blk *
 364 readblock(int fd, u_int32_t clstsize)
 365 {
 366         int numread;
 367         struct mkuz_blk *rval;
 368         static int blockcnt;
 369         off_t cpos;
 370
 371         rval = mkuz_blk_ctor(clstsize);
 372
 373         rval->info.blkno = blockcnt;
 374         blockcnt += 1;
 375         cpos = lseek(fd, 0, SEEK_CUR);
 376         if (cpos < 0) {
 377                 err(1, "readblock: lseek() failed");
 378                 /* Not reached */
 379         }
 380         rval->info.offset = cpos;
 381
 382         numread = read(fd, rval->data, clstsize);
 383         if (numread < 0) {
 384                 err(1, "readblock: read() failed");
 385                 /* Not reached */
 386         }
 387         if (numread == 0) {
 388                 free(rval);
 389                 return MKUZ_BLK_EOF;
 390         }
 391         rval->info.len = numread;
 392         return rval;
 393 }
 394
 395 static void
 396 usage(void)
 397 {
 398
 399         fprintf(stderr, "usage: mkuzip [-vZdLS] [-o outfile] [-s cluster_size] "
 400             "[-j ncompr] infile\n");
 401         exit(1);
 402 }
 403
 404 void *
 405 mkuz_safe_malloc(size_t size)
 406 {
 407         void *retval;
 408
 409         retval = malloc(size);
 410         if (retval == NULL) {
 411                 err(1, "can't allocate memory");
 412                 /* Not reached */
 413         }
 414         return retval;
 415 }
 416
 417 void *
 418 mkuz_safe_zmalloc(size_t size)
 419 {
 420         void *retval;
 421
 422         retval = mkuz_safe_malloc(size);
 423         bzero(retval, size);
 424         return retval;
 425 }
 426
 427 static void
 428 cleanup(void)
 429 {
 430
 431         if (cleanfile != NULL)
 432                 unlink(cleanfile);
 433 }
 434
 435 int
 436 mkuz_memvcmp(const void *memory, unsigned char val, size_t size)
 437 {
 438     const u_char *mm;
 439
 440     mm = (const u_char *)memory;
 441     return (*mm == val) && memcmp(mm, mm + 1, size - 1) == 0;
 442 }