2 * SPDX-License-Identifier: BSD-3-Clause
4 * Copyright (c) 1985, 1986, 1992, 1993
5 * The Regents of the University of California. All rights reserved.
7 * This code is derived from software contributed to Berkeley by
8 * Diomidis Spinellis and James A. Woods, derived from original
9 * work by Spencer Thomas and Joseph Orost.
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. Neither the name of the University nor the names of its contributors
20 * may be used to endorse or promote products derived from this software
21 * without specific prior written permission.
23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 #if defined(LIBC_SCCS) && !defined(lint)
37 static char sccsid[] = "@(#)zopen.c 8.1 (Berkeley) 6/27/93";
38 #endif /* LIBC_SCCS and not lint */
40 #include <sys/cdefs.h>
41 __FBSDID("$FreeBSD$");
44 * fcompress.c - File compression ala IEEE Computer, June 1984.
47 * Spencer W. Thomas (decvax!utah-cs!thomas)
48 * Jim McKie (decvax!mcvax!jim)
49 * Steve Davies (decvax!vax135!petsd!peora!srd)
50 * Ken Turkowski (decvax!decwrl!turtlevax!ken)
51 * James A. Woods (decvax!ihnp4!ames!jaw)
52 * Joe Orost (decvax!vax135!petsd!joe)
54 * Cleaned up and converted to library returning I/O streams by
55 * Diomidis Spinellis <dds@doc.ic.ac.uk>.
57 * zopen(filename, mode, bits)
58 * Returns a FILE * that can be used for read or write. The modes
59 * supported are only "r" and "w". Seeking is not allowed. On
60 * reading the file is decompressed, on writing it is compressed.
61 * The output is compatible with compress(1) with 16 bit tables.
62 * Any file produced by compress(1) can be read.
65 #include <sys/param.h>
77 #define BITS 16 /* Default bits. */
78 #define HSIZE 69001 /* 95% occupancy */
80 /* A code_int must be able to hold 2**BITS values of type int, and also -1. */
81 typedef long code_int;
82 typedef long count_int;
84 typedef u_char char_type;
85 static char_type magic_header[] =
86 {'\037', '\235'}; /* 1F 9D */
88 #define BIT_MASK 0x1f /* Defines for third byte of header. */
89 #define BLOCK_MASK 0x80
92 * Masks 0x40 and 0x20 are free. I think 0x20 should mean that there is
93 * a fourth header byte (for expansion).
95 #define INIT_BITS 9 /* Initial number of bits/code. */
97 #define MAXCODE(n_bits) ((1 << (n_bits)) - 1)
100 FILE *zs_fp; /* File stream for I/O */
101 char zs_mode; /* r or w */
103 S_START, S_MIDDLE, S_EOF
104 } zs_state; /* State of computation */
105 u_int zs_n_bits; /* Number of bits/code. */
106 u_int zs_maxbits; /* User settable max # bits/code. */
107 code_int zs_maxcode; /* Maximum code, given n_bits. */
108 code_int zs_maxmaxcode; /* Should NEVER generate this code. */
109 count_int zs_htab [HSIZE];
110 u_short zs_codetab [HSIZE];
111 code_int zs_hsize; /* For dynamic table sizing. */
112 code_int zs_free_ent; /* First unused entry. */
114 * Block compression parameters -- after all codes are used up,
115 * and compression rate changes, start over.
117 int zs_block_compress;
120 count_int zs_checkpoint;
122 long zs_in_count; /* Length of input. */
123 long zs_bytes_out; /* Length of compressed output. */
124 long zs_out_count; /* # of codes output (for debugging). */
125 char_type zs_buf[BITS];
130 code_int zs_hsize_reg;
132 } w; /* Write parameters */
134 char_type *zs_stackp;
136 code_int zs_code, zs_oldcode, zs_incode;
137 int zs_roffset, zs_size;
138 char_type zs_gbuf[BITS];
139 } r; /* Read parameters */
143 /* Definitions to retain old variable names */
145 #define zmode zs->zs_mode
146 #define state zs->zs_state
147 #define n_bits zs->zs_n_bits
148 #define maxbits zs->zs_maxbits
149 #define maxcode zs->zs_maxcode
150 #define maxmaxcode zs->zs_maxmaxcode
151 #define htab zs->zs_htab
152 #define codetab zs->zs_codetab
153 #define hsize zs->zs_hsize
154 #define free_ent zs->zs_free_ent
155 #define block_compress zs->zs_block_compress
156 #define clear_flg zs->zs_clear_flg
157 #define ratio zs->zs_ratio
158 #define checkpoint zs->zs_checkpoint
159 #define offset zs->zs_offset
160 #define in_count zs->zs_in_count
161 #define bytes_out zs->zs_bytes_out
162 #define out_count zs->zs_out_count
163 #define buf zs->zs_buf
164 #define fcode zs->u.w.zs_fcode
165 #define hsize_reg zs->u.w.zs_hsize_reg
166 #define ent zs->u.w.zs_ent
167 #define hshift zs->u.w.zs_hshift
168 #define stackp zs->u.r.zs_stackp
169 #define finchar zs->u.r.zs_finchar
170 #define code zs->u.r.zs_code
171 #define oldcode zs->u.r.zs_oldcode
172 #define incode zs->u.r.zs_incode
173 #define roffset zs->u.r.zs_roffset
174 #define size zs->u.r.zs_size
175 #define gbuf zs->u.r.zs_gbuf
178 * To save much memory, we overlay the table used by compress() with those
179 * used by decompress(). The tab_prefix table is the same size and type as
180 * the codetab. The tab_suffix table needs 2**BITS characters. We get this
181 * from the beginning of htab. The output stack uses the rest of htab, and
182 * contains characters. There is plenty of room for any possible stack
183 * (stack used to be 8000 characters).
186 #define htabof(i) htab[i]
187 #define codetabof(i) codetab[i]
189 #define tab_prefixof(i) codetabof(i)
190 #define tab_suffixof(i) ((char_type *)(htab))[i]
191 #define de_stack ((char_type *)&tab_suffixof(1 << BITS))
193 #define CHECK_GAP 10000 /* Ratio check interval. */
196 * the next two codes should not be changed lightly, as they must not
197 * lie within the contiguous general code space.
199 #define FIRST 257 /* First free entry. */
200 #define CLEAR 256 /* Table clear output code. */
202 static int cl_block(struct s_zstate *);
203 static void cl_hash(struct s_zstate *, count_int);
204 static code_int getcode(struct s_zstate *);
205 static int output(struct s_zstate *, code_int);
206 static int zclose(void *);
207 static int zread(void *, char *, int);
208 static int zwrite(void *, const char *, int);
211 * Algorithm from "A Technique for High Performance Data Compression",
212 * Terry A. Welch, IEEE Computer Vol 17, No 6 (June 1984), pp 8-19.
215 * Modified Lempel-Ziv method (LZW). Basically finds common
216 * substrings and replaces them with a variable size code. This is
217 * deterministic, and can be done on the fly. Thus, the decompression
218 * procedure needs no input table, but tracks the way the table was built.
224 * Algorithm: use open addressing double hashing (no chaining) on the
225 * prefix code / next character combination. We do a variant of Knuth's
226 * algorithm D (vol. 3, sec. 6.4) along with G. Knott's relatively-prime
227 * secondary probe. Here, the modular division first probe is gives way
228 * to a faster exclusive-or manipulation. Also do block compression with
229 * an adaptive reset, whereby the code table is cleared when the compression
230 * ratio decreases, but after the table fills. The variable-length output
231 * codes are re-sized at this point, and a special CLEAR code is generated
232 * for the decompressor. Late addition: construct the table according to
233 * file size for noticeable speed improvement on small files. Please direct
234 * questions about this implementation to ames!jaw.
237 zwrite(void *cookie, const char *wbp, int num)
251 bp = (const u_char *)wbp;
252 if (state == S_MIDDLE)
256 maxmaxcode = 1L << maxbits;
257 if (fwrite(magic_header,
258 sizeof(char), sizeof(magic_header), fp) != sizeof(magic_header))
260 tmp = (u_char)((maxbits) | block_compress);
261 if (fwrite(&tmp, sizeof(char), sizeof(tmp), fp) != sizeof(tmp))
265 bytes_out = 3; /* Includes 3-byte header mojo. */
270 checkpoint = CHECK_GAP;
271 maxcode = MAXCODE(n_bits = INIT_BITS);
272 free_ent = ((block_compress) ? FIRST : 256);
278 for (fcode = (long)hsize; fcode < 65536L; fcode *= 2L)
280 hshift = 8 - hshift; /* Set hash code range bound. */
283 cl_hash(zs, (count_int)hsize_reg); /* Clear hash table. */
285 middle: for (i = 0; count--;) {
288 fcode = (long)(((long)c << maxbits) + ent);
289 i = ((c << hshift) ^ ent); /* Xor hashing. */
291 if (htabof(i) == fcode) {
294 } else if ((long)htabof(i) < 0) /* Empty slot. */
296 disp = hsize_reg - i; /* Secondary hash (after G. Knott). */
299 probe: if ((i -= disp) < 0)
302 if (htabof(i) == fcode) {
306 if ((long)htabof(i) >= 0)
308 nomatch: if (output(zs, (code_int) ent) == -1)
312 if (free_ent < maxmaxcode) {
313 codetabof(i) = free_ent++; /* code -> hashtable */
315 } else if ((count_int)in_count >=
316 checkpoint && block_compress) {
317 if (cl_block(zs) == -1)
331 if (zmode == 'w') { /* Put out the final code. */
332 if (output(zs, (code_int) ent) == -1) {
338 if (output(zs, (code_int) - 1) == -1) {
344 rval = fclose(fp) == EOF ? -1 : 0;
350 * Output the given code.
352 * code: A n_bits-bit integer. If == -1, then EOF. This assumes
353 * that n_bits =< (long)wordsize - 1.
355 * Outputs code to the file.
357 * Chars are 8 bits long.
359 * Maintain a BITS character long buffer (so that 8 codes will
360 * fit in it exactly). Use the VAX insv instruction to insert each
361 * code in turn. When the buffer fills up empty it and start over.
364 static char_type lmask[9] =
365 {0xff, 0xfe, 0xfc, 0xf8, 0xf0, 0xe0, 0xc0, 0x80, 0x00};
366 static char_type rmask[9] =
367 {0x00, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff};
370 output(struct s_zstate *zs, code_int ocode)
380 /* Get to the first byte. */
384 * Since ocode is always >= 8 bits, only need to mask the first
387 *bp = (*bp & rmask[r_off]) | ((ocode << r_off) & lmask[r_off]);
391 /* Get any 8 bit parts in the middle (<=1 for up to 16 bits). */
401 if (offset == (n_bits << 3)) {
405 if (fwrite(bp, sizeof(char), bits, fp) != bits)
412 * If the next entry is going to be too big for the ocode size,
413 * then increase it, if possible.
415 if (free_ent > maxcode || (clear_flg > 0)) {
417 * Write the whole buffer, because the input side won't
418 * discover the size increase until after it has read it.
421 if (fwrite(buf, 1, n_bits, fp) != n_bits)
428 maxcode = MAXCODE(n_bits = INIT_BITS);
432 if (n_bits == maxbits)
433 maxcode = maxmaxcode;
435 maxcode = MAXCODE(n_bits);
439 /* At EOF, write the rest of the buffer. */
441 offset = (offset + 7) / 8;
442 if (fwrite(buf, 1, offset, fp) != offset)
452 * Decompress read. This routine adapts to the codes in the file building
453 * the "string" table on-the-fly; requiring no table to be stored in the
454 * compressed file. The tables used herein are shared with those of the
455 * compress() routine. See the definitions above.
458 zread(void *cookie, char *rbp, int num)
462 u_char *bp, header[3];
480 /* Check the magic number */
482 sizeof(char), sizeof(header), fp) != sizeof(header) ||
483 memcmp(header, magic_header, sizeof(magic_header)) != 0) {
487 maxbits = header[2]; /* Set -b from file. */
488 block_compress = maxbits & BLOCK_MASK;
490 maxmaxcode = 1L << maxbits;
491 if (maxbits > BITS || maxbits < 12) {
495 /* As above, initialize the first 256 entries in the table. */
496 maxcode = MAXCODE(n_bits = INIT_BITS);
497 for (code = 255; code >= 0; code--) {
498 tab_prefixof(code) = 0;
499 tab_suffixof(code) = (char_type) code;
501 free_ent = block_compress ? FIRST : 256;
503 finchar = oldcode = getcode(zs);
504 if (oldcode == -1) /* EOF already? */
505 return (0); /* Get out of here */
507 /* First code must be 8 bits = char. */
508 *bp++ = (u_char)finchar;
512 while ((code = getcode(zs)) > -1) {
514 if ((code == CLEAR) && block_compress) {
515 for (code = 255; code >= 0; code--)
516 tab_prefixof(code) = 0;
524 /* Special case for kWkWk string. */
525 if (code >= free_ent) {
526 if (code > free_ent || oldcode == -1) {
535 * The above condition ensures that code < free_ent.
536 * The construction of tab_prefixof in turn guarantees that
537 * each iteration decreases code and therefore stack usage is
538 * bound by 1 << BITS - 256.
541 /* Generate output characters in reverse order. */
542 while (code >= 256) {
543 *stackp++ = tab_suffixof(code);
544 code = tab_prefixof(code);
546 *stackp++ = finchar = tab_suffixof(code);
548 /* And put them out in forward order. */
553 } while (stackp > de_stack);
555 /* Generate the new entry. */
556 if ((code = free_ent) < maxmaxcode && oldcode != -1) {
557 tab_prefixof(code) = (u_short) oldcode;
558 tab_suffixof(code) = finchar;
562 /* Remember previous code. */
566 eof: return (num - count);
570 * Read one code from the standard input. If EOF, return -1.
574 * code or -1 is returned.
577 getcode(struct s_zstate *zs)
584 if (clear_flg > 0 || roffset >= size || free_ent > maxcode) {
586 * If the next entry will be too big for the current gcode
587 * size, then we must increase the size. This implies reading
588 * a new buffer full, too.
590 if (free_ent > maxcode) {
592 if (n_bits == maxbits) /* Won't get any bigger now. */
593 maxcode = maxmaxcode;
595 maxcode = MAXCODE(n_bits);
598 maxcode = MAXCODE(n_bits = INIT_BITS);
601 size = fread(gbuf, 1, n_bits, fp);
602 if (size <= 0) /* End of file. */
605 /* Round size down to integral number of codes. */
606 size = (size << 3) - (n_bits - 1);
611 /* Get to the first byte. */
615 /* Get first part (low order bits). */
616 gcode = (*bp++ >> r_off);
618 r_off = 8 - r_off; /* Now, roffset into gcode word. */
620 /* Get any 8 bit parts in the middle (<=1 for up to 16 bits). */
622 gcode |= *bp++ << r_off;
627 /* High order bits. */
628 gcode |= (*bp & rmask[bits]) << r_off;
635 cl_block(struct s_zstate *zs) /* Table clear for block compress. */
639 checkpoint = in_count + CHECK_GAP;
641 if (in_count > 0x007fffff) { /* Shift will overflow. */
642 rat = bytes_out >> 8;
643 if (rat == 0) /* Don't divide by zero. */
646 rat = in_count / rat;
648 rat = (in_count << 8) / bytes_out; /* 8 fractional bits. */
653 cl_hash(zs, (count_int) hsize);
656 if (output(zs, (code_int) CLEAR) == -1)
663 cl_hash(struct s_zstate *zs, count_int cl_hsize) /* Reset code table. */
669 htab_p = htab + cl_hsize;
671 do { /* Might use Sys V memset(3) here. */
689 } while ((i -= 16) >= 0);
690 for (i += 16; i > 0; i--)
695 zopen(const char *fname, const char *mode, int bits)
699 if ((mode[0] != 'r' && mode[0] != 'w') || mode[1] != '\0' ||
700 bits < 0 || bits > BITS) {
705 if ((zs = calloc(1, sizeof(struct s_zstate))) == NULL)
708 maxbits = bits ? bits : BITS; /* User settable max # bits/code. */
709 maxmaxcode = 1L << maxbits; /* Should NEVER generate this code. */
710 hsize = HSIZE; /* For dynamic table sizing. */
711 free_ent = 0; /* First unused entry. */
712 block_compress = BLOCK_MASK;
715 checkpoint = CHECK_GAP;
716 in_count = 1; /* Length of input. */
717 out_count = 0; /* # of codes output (for debugging). */
723 * Layering compress on top of stdio in order to provide buffering,
724 * and ensure that reads and write work with the data specified.
726 if ((fp = fopen(fname, mode)) == NULL) {
733 return (funopen(zs, zread, NULL, NULL, zclose));
736 return (funopen(zs, NULL, zwrite, NULL, zclose));