1 /* $Id: read.c,v 1.131 2015/03/11 13:05:20 schwarze Exp $ */
3 * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv>
4 * Copyright (c) 2010-2015 Ingo Schwarze <schwarze@openbsd.org>
5 * Copyright (c) 2010, 2012 Joerg Sonnenberger <joerg@netbsd.org>
7 * Permission to use, copy, modify, and distribute this software for any
8 * purpose with or without fee is hereby granted, provided that the above
9 * copyright notice and this permission notice appear in all copies.
11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
21 #include <sys/types.h>
42 #include "mandoc_aux.h"
43 #include "libmandoc.h"
47 #define REPARSE_LIMIT 1000
50 struct man *pman; /* persistent man parser */
51 struct mdoc *pmdoc; /* persistent mdoc parser */
52 struct man *man; /* man parser */
53 struct mdoc *mdoc; /* mdoc parser */
54 struct roff *roff; /* roff parser (!NULL) */
55 const struct mchars *mchars; /* character table */
56 char *sodest; /* filename pointed to by .so */
57 const char *file; /* filename of current input file */
58 struct buf *primary; /* buffer currently being parsed */
59 struct buf *secondary; /* preprocessed copy of input */
60 const char *defos; /* default operating system */
61 mandocmsg mmsg; /* warning/error message handler */
62 enum mandoclevel file_status; /* status of current parse */
63 enum mandoclevel wlevel; /* ignore messages below this */
64 int options; /* parser options */
65 int filenc; /* encoding of the current file */
66 int reparse_count; /* finite interp. stack */
67 int line; /* line number in the file */
68 pid_t child; /* the gunzip(1) process */
71 static void choose_parser(struct mparse *);
72 static void resize_buf(struct buf *, size_t);
73 static void mparse_buf_r(struct mparse *, struct buf, size_t, int);
74 static int read_whole_file(struct mparse *, const char *, int,
76 static void mparse_end(struct mparse *);
77 static void mparse_parse_buffer(struct mparse *, struct buf,
80 static const enum mandocerr mandoclimits[MANDOCLEVEL_MAX] = {
90 static const char * const mandocerrs[MANDOCERR_MAX] = {
95 /* related to the prologue */
96 "missing manual title, using UNTITLED",
97 "missing manual title, using \"\"",
98 "lower case character in document title",
99 "missing manual section, using \"\"",
100 "unknown manual section",
101 "missing date, using today's date",
102 "cannot parse date, using it verbatim",
103 "missing Os macro, using \"\"",
104 "duplicate prologue macro",
105 "late prologue macro",
106 "skipping late title macro",
107 "prologue macros out of order",
109 /* related to document structure */
110 ".so is fragile, better use ln(1)",
112 "content before first section header",
113 "first section is not \"NAME\"",
114 "NAME section without name",
115 "NAME section without description",
116 "description not at the end of NAME",
117 "bad NAME section content",
118 "missing description line, using \"\"",
119 "sections out of conventional order",
120 "duplicate section title",
121 "unexpected section",
123 "unusual Xr punctuation",
124 "AUTHORS section without An macro",
126 /* related to macros and nesting */
128 "macro neither callable nor escaped",
129 "skipping paragraph macro",
130 "moving paragraph macro out of list",
131 "skipping no-space macro",
132 "blocks badly nested",
133 "nested displays are not portable",
134 "moving content out of list",
135 ".Vt block has child macro",
136 "fill mode already enabled, skipping",
137 "fill mode already disabled, skipping",
140 /* related to missing macro arguments */
141 "skipping empty request",
142 "conditional request controls empty scope",
143 "skipping empty macro",
145 "empty argument, using 0n",
146 "missing display type, using -ragged",
147 "list type is not the first argument",
148 "missing -width in -tag list, using 8n",
149 "missing utility name, using \"\"",
150 "missing function name, using \"\"",
151 "empty head in list item",
153 "missing font type, using \\fR",
154 "unknown font type, using \\fR",
155 "nothing follows prefix",
156 "empty reference block",
157 "missing -std argument, adding it",
158 "missing option string, using \"\"",
159 "missing resource identifier, using \"\"",
160 "missing eqn box, using \"\"",
162 /* related to bad macro arguments */
163 "unterminated quoted argument",
164 "duplicate argument",
165 "skipping duplicate argument",
166 "skipping duplicate display type",
167 "skipping duplicate list type",
168 "skipping -width argument",
169 "wrong number of cells",
170 "unknown AT&T UNIX version",
171 "comma in function argument",
172 "parenthesis in function name",
173 "invalid content in Rs block",
174 "invalid Boolean argument",
175 "unknown font, skipping request",
176 "odd number of characters in request",
178 /* related to plain text */
179 "blank line in fill mode, using .sp",
180 "tab in filled text",
181 "whitespace at end of input line",
183 "invalid escape sequence",
184 "undefined string, using \"\"",
186 /* related to tables */
187 "tbl line starts with span",
188 "tbl column starts with span",
189 "skipping vertical bar in tbl layout",
193 /* related to tables */
194 "non-alphabetic character in tbl options",
195 "skipping unknown tbl option",
196 "missing tbl option argument",
197 "wrong tbl option argument size",
199 "invalid character in tbl layout",
200 "unmatched parenthesis in tbl layout",
201 "tbl without any data cells",
202 "ignoring data in spanned tbl cell",
203 "ignoring extra tbl data cells",
204 "data block open at end of tbl",
206 /* related to document structure and macros */
208 "input stack limit exceeded, infinite loop?",
209 "skipping bad character",
210 "skipping unknown macro",
211 "skipping insecure request",
212 "skipping item outside list",
213 "skipping column outside column list",
214 "skipping end of block that is not open",
215 "fewer RS blocks open, skipping",
216 "inserting missing end of block",
217 "appending missing end of block",
219 /* related to request and macro arguments */
220 "escaped character not allowed in a name",
221 "NOT IMPLEMENTED: Bd -file",
222 "missing list type, using -item",
223 "missing manual name, using \"\"",
224 "uname(3) system call failed, using UNKNOWN",
225 "unknown standard specifier",
226 "skipping request without numeric argument",
227 "NOT IMPLEMENTED: .so with absolute path or \"..\"",
228 ".so request failed",
229 "skipping all arguments",
230 "skipping excess arguments",
233 "unsupported feature",
235 "unsupported control character",
236 "unsupported roff request",
237 "eqn delim option in tbl",
238 "unsupported tbl layout modifier",
239 "ignoring macro in table",
242 static const char * const mandoclevels[MANDOCLEVEL_MAX] = {
254 resize_buf(struct buf *buf, size_t initial)
257 buf->sz = buf->sz > initial/2 ? 2 * buf->sz : initial;
258 buf->buf = mandoc_realloc(buf->buf, buf->sz);
262 choose_parser(struct mparse *curp)
268 * If neither command line arguments -mdoc or -man select
269 * a parser nor the roff parser found a .Dd or .TH macro
270 * yet, look ahead in the main input buffer.
273 if ((format = roff_getformat(curp->roff)) == 0) {
274 cp = curp->primary->buf;
275 ep = cp + curp->primary->sz;
277 if (*cp == '.' || *cp == '\'') {
279 if (cp[0] == 'D' && cp[1] == 'd') {
280 format = MPARSE_MDOC;
283 if (cp[0] == 'T' && cp[1] == 'H') {
288 cp = memchr(cp, '\n', ep - cp);
295 if (format == MPARSE_MDOC) {
296 if (NULL == curp->pmdoc)
297 curp->pmdoc = mdoc_alloc(
298 curp->roff, curp, curp->defos,
299 MPARSE_QUICK & curp->options ? 1 : 0);
301 curp->mdoc = curp->pmdoc;
305 /* Fall back to man(7) as a last resort. */
307 if (NULL == curp->pman)
308 curp->pman = man_alloc(
309 curp->roff, curp, curp->defos,
310 MPARSE_QUICK & curp->options ? 1 : 0);
312 curp->man = curp->pman;
316 * Main parse routine for a buffer.
317 * It assumes encoding and line numbering are already set up.
318 * It can recurse directly (for invocations of user-defined
319 * macros, inline equations, and input line traps)
320 * and indirectly (for .so file inclusion).
323 mparse_buf_r(struct mparse *curp, struct buf blk, size_t i, int start)
325 const struct tbl_span *span;
327 const char *save_file;
329 size_t pos; /* byte number in the ln buffer */
332 int lnn; /* line number in the real file */
337 memset(&ln, 0, sizeof(ln));
343 if (0 == pos && '\0' == blk.buf[i])
348 curp->reparse_count = 0;
351 curp->filenc & MPARSE_UTF8 &&
352 curp->filenc & MPARSE_LATIN1)
353 curp->filenc = preconv_cue(&blk, i);
356 while (i < blk.sz && (start || blk.buf[i] != '\0')) {
359 * When finding an unescaped newline character,
360 * leave the character loop to process the line.
361 * Skip a preceding carriage return, if any.
364 if ('\r' == blk.buf[i] && i + 1 < blk.sz &&
365 '\n' == blk.buf[i + 1])
367 if ('\n' == blk.buf[i]) {
374 * Make sure we have space for the worst
375 * case of 11 bytes: "\\[u10ffff]\0"
378 if (pos + 11 > ln.sz)
379 resize_buf(&ln, 256);
382 * Encode 8-bit input.
387 if ( ! (curp->filenc && preconv_encode(
388 &blk, &i, &ln, &pos, &curp->filenc))) {
389 mandoc_vmsg(MANDOCERR_CHAR_BAD, curp,
390 curp->line, pos, "0x%x", c);
398 * Exclude control characters.
401 if (c == 0x7f || (c < 0x20 && c != 0x09)) {
402 mandoc_vmsg(c == 0x00 || c == 0x04 ||
403 c > 0x0a ? MANDOCERR_CHAR_BAD :
404 MANDOCERR_CHAR_UNSUPP,
405 curp, curp->line, pos, "0x%x", c);
412 /* Trailing backslash = a plain char. */
414 if (blk.buf[i] != '\\' || i + 1 == blk.sz) {
415 ln.buf[pos++] = blk.buf[i++];
420 * Found escape and at least one other character.
421 * When it's a newline character, skip it.
422 * When there is a carriage return in between,
423 * skip that one as well.
426 if ('\r' == blk.buf[i + 1] && i + 2 < blk.sz &&
427 '\n' == blk.buf[i + 2])
429 if ('\n' == blk.buf[i + 1]) {
435 if ('"' == blk.buf[i + 1] || '#' == blk.buf[i + 1]) {
437 /* Comment, skip to end of line */
438 for (; i < blk.sz; ++i) {
439 if ('\n' == blk.buf[i]) {
446 /* Backout trailing whitespaces */
447 for (; pos > 0; --pos) {
448 if (ln.buf[pos - 1] != ' ')
450 if (pos > 2 && ln.buf[pos - 2] == '\\')
456 /* Catch escaped bogus characters. */
458 c = (unsigned char) blk.buf[i+1];
460 if ( ! (isascii(c) &&
461 (isgraph(c) || isblank(c)))) {
462 mandoc_vmsg(MANDOCERR_CHAR_BAD, curp,
463 curp->line, pos, "0x%x", c);
469 /* Some other escape sequence, copy & cont. */
471 ln.buf[pos++] = blk.buf[i++];
472 ln.buf[pos++] = blk.buf[i++];
476 resize_buf(&ln, 256);
481 * A significant amount of complexity is contained by
482 * the roff preprocessor. It's line-oriented but can be
483 * expressed on one line, so we need at times to
484 * readjust our starting point and re-run it. The roff
485 * preprocessor can also readjust the buffers with new
486 * data, so we pass them in wholesale.
492 * Maintain a lookaside buffer of all parsed lines. We
493 * only do this if mparse_keep() has been invoked (the
494 * buffer may be accessed with mparse_getkeep()).
497 if (curp->secondary) {
498 curp->secondary->buf = mandoc_realloc(
499 curp->secondary->buf,
500 curp->secondary->sz + pos + 2);
501 memcpy(curp->secondary->buf +
504 curp->secondary->sz += pos;
506 [curp->secondary->sz] = '\n';
507 curp->secondary->sz++;
509 [curp->secondary->sz] = '\0';
512 rr = roff_parseln(curp->roff, curp->line, &ln, &of);
516 if (REPARSE_LIMIT >= ++curp->reparse_count)
517 mparse_buf_r(curp, ln, of, 0);
519 mandoc_msg(MANDOCERR_ROFFLOOP, curp,
520 curp->line, pos, NULL);
524 pos = strlen(ln.buf);
532 if ( ! (curp->options & MPARSE_SO) &&
533 (i >= blk.sz || blk.buf[i] == '\0')) {
534 curp->sodest = mandoc_strdup(ln.buf + of);
539 * We remove `so' clauses from our lookaside
540 * buffer because we're going to descend into
541 * the file recursively.
544 curp->secondary->sz -= pos + 1;
545 save_file = curp->file;
546 save_child = curp->child;
547 if (mparse_open(curp, &fd, ln.buf + of) ==
549 mparse_readfd(curp, fd, ln.buf + of);
550 curp->file = save_file;
552 curp->file = save_file;
553 mandoc_vmsg(MANDOCERR_SO_FAIL,
554 curp, curp->line, pos,
555 ".so %s", ln.buf + of);
556 ln.sz = mandoc_asprintf(&cp,
557 ".sp\nSee the file %s.\n.sp",
562 mparse_buf_r(curp, ln, of, 0);
564 curp->child = save_child;
572 * If input parsers have not been allocated, do so now.
573 * We keep these instanced between parsers, but set them
574 * locally per parse routine since we can use different
575 * parsers with each one.
578 if ( ! (curp->man || curp->mdoc))
582 * Lastly, push down into the parsers themselves.
583 * If libroff returns ROFF_TBL, then add it to the
584 * currently open parse. Since we only get here if
585 * there does exist data (see tbl_data.c), we're
586 * guaranteed that something's been allocated.
587 * Do the same for ROFF_EQN.
590 if (rr == ROFF_TBL) {
591 while ((span = roff_span(curp->roff)) != NULL)
592 if (curp->man == NULL)
593 mdoc_addspan(curp->mdoc, span);
595 man_addspan(curp->man, span);
596 } else if (rr == ROFF_EQN) {
597 if (curp->man == NULL)
598 mdoc_addeqn(curp->mdoc, roff_eqn(curp->roff));
600 man_addeqn(curp->man, roff_eqn(curp->roff));
601 } else if ((curp->man == NULL ?
602 mdoc_parseln(curp->mdoc, curp->line, ln.buf, of) :
603 man_parseln(curp->man, curp->line, ln.buf, of)) == 2)
606 /* Temporary buffers typically are not full. */
608 if (0 == start && '\0' == blk.buf[i])
611 /* Start the next input line. */
620 read_whole_file(struct mparse *curp, const char *file, int fd,
621 struct buf *fb, int *with_mmap)
628 if (-1 == fstat(fd, &st)) {
630 exit((int)MANDOCLEVEL_SYSERR);
634 * If we're a regular file, try just reading in the whole entry
635 * via mmap(). This is faster than reading it into blocks, and
636 * since each file is only a few bytes to begin with, I'm not
637 * concerned that this is going to tank any machines.
640 if (S_ISREG(st.st_mode)) {
641 if (st.st_size > 0x7fffffff) {
642 mandoc_msg(MANDOCERR_TOOLARGE, curp, 0, 0, NULL);
646 fb->sz = (size_t)st.st_size;
647 fb->buf = mmap(NULL, fb->sz, PROT_READ, MAP_SHARED, fd, 0);
648 if (fb->buf != MAP_FAILED)
654 * If this isn't a regular file (like, say, stdin), then we must
655 * go the old way and just read things in bit by bit.
664 if (fb->sz == (1U << 31)) {
665 mandoc_msg(MANDOCERR_TOOLARGE, curp,
669 resize_buf(fb, 65536);
671 ssz = read(fd, fb->buf + (int)off, fb->sz - off);
678 exit((int)MANDOCLEVEL_SYSERR);
689 mparse_end(struct mparse *curp)
692 if (curp->mdoc == NULL &&
694 curp->sodest == NULL) {
695 if (curp->options & MPARSE_MDOC)
696 curp->mdoc = curp->pmdoc;
698 if (curp->pman == NULL)
699 curp->pman = man_alloc(
700 curp->roff, curp, curp->defos,
701 curp->options & MPARSE_QUICK ? 1 : 0);
702 curp->man = curp->pman;
706 mdoc_endparse(curp->mdoc);
708 man_endparse(curp->man);
709 roff_endparse(curp->roff);
713 mparse_parse_buffer(struct mparse *curp, struct buf blk, const char *file)
715 struct buf *svprimary;
718 static int recursion_depth;
720 if (64 < recursion_depth) {
721 mandoc_msg(MANDOCERR_ROFFLOOP, curp, curp->line, 0, NULL);
725 /* Line number is per-file. */
728 svprimary = curp->primary;
729 curp->primary = &blk;
733 /* Skip an UTF-8 byte order mark. */
734 if (curp->filenc & MPARSE_UTF8 && blk.sz > 2 &&
735 (unsigned char)blk.buf[0] == 0xef &&
736 (unsigned char)blk.buf[1] == 0xbb &&
737 (unsigned char)blk.buf[2] == 0xbf) {
739 curp->filenc &= ~MPARSE_LATIN1;
743 mparse_buf_r(curp, blk, offset, 1);
745 if (--recursion_depth == 0)
748 curp->primary = svprimary;
753 mparse_readmem(struct mparse *curp, void *buf, size_t len,
761 mparse_parse_buffer(curp, blk, file);
762 return(curp->file_status);
766 * Read the whole file into memory and call the parsers.
767 * Called recursively when an .so request is encountered.
770 mparse_readfd(struct mparse *curp, int fd, const char *file)
776 if (read_whole_file(curp, file, fd, &blk, &with_mmap)) {
777 save_filenc = curp->filenc;
778 curp->filenc = curp->options &
779 (MPARSE_UTF8 | MPARSE_LATIN1);
780 mparse_parse_buffer(curp, blk, file);
781 curp->filenc = save_filenc;
784 munmap(blk.buf, blk.sz);
790 if (fd != STDIN_FILENO && close(fd) == -1)
794 return(curp->file_status);
798 * hack to avoid depending on gnuzip(1) waiting for upstream proper
802 gunzip(const char *file)
808 gz = gzopen(file, "r");
810 err(EXIT_FAILURE, "cannot open %s", file);
812 while ((r = gzread(gz, buf, sizeof(buf))) > 0)
813 fwrite(buf, 1, r, stdout);
816 return (EXIT_SUCCESS);
819 mparse_open(struct mparse *curp, int *fd, const char *file)
827 /* Unless zipped, try to just open the file. */
829 if ((cp = strrchr(file, '.')) == NULL ||
830 strcmp(cp + 1, "gz")) {
832 if ((*fd = open(file, O_RDONLY)) != -1)
833 return(MANDOCLEVEL_OK);
835 /* Open failed; try to append ".gz". */
837 mandoc_asprintf(&cp, "%s.gz", file);
842 /* Before forking, make sure the file can be read. */
845 if (access(file, R_OK) == -1) {
851 mandoc_msg(MANDOCERR_FILE, curp, 0, 0, strerror(errno));
852 return(MANDOCLEVEL_ERROR);
857 if (pipe(pfd) == -1) {
859 exit((int)MANDOCLEVEL_SYSERR);
862 switch (curp->child = fork()) {
865 exit((int)MANDOCLEVEL_SYSERR);
868 if (dup2(pfd[1], STDOUT_FILENO) == -1) {
870 exit((int)MANDOCLEVEL_SYSERR);
876 return(MANDOCLEVEL_OK);
881 mparse_wait(struct mparse *curp)
885 if (curp->child == 0)
886 return(MANDOCLEVEL_OK);
888 if (waitpid(curp->child, &status, 0) == -1) {
890 exit((int)MANDOCLEVEL_SYSERR);
893 if (WIFSIGNALED(status)) {
894 mandoc_vmsg(MANDOCERR_FILE, curp, 0, 0,
895 "gunzip died from signal %d", WTERMSIG(status));
896 return(MANDOCLEVEL_ERROR);
898 if (WEXITSTATUS(status)) {
899 mandoc_vmsg(MANDOCERR_FILE, curp, 0, 0,
900 "gunzip failed with code %d", WEXITSTATUS(status));
901 return(MANDOCLEVEL_ERROR);
903 return(MANDOCLEVEL_OK);
907 mparse_alloc(int options, enum mandoclevel wlevel, mandocmsg mmsg,
908 const struct mchars *mchars, const char *defos)
912 curp = mandoc_calloc(1, sizeof(struct mparse));
914 curp->options = options;
915 curp->wlevel = wlevel;
919 curp->mchars = mchars;
920 curp->roff = roff_alloc(curp, curp->mchars, options);
921 if (curp->options & MPARSE_MDOC)
922 curp->pmdoc = mdoc_alloc(
923 curp->roff, curp, curp->defos,
924 curp->options & MPARSE_QUICK ? 1 : 0);
925 if (curp->options & MPARSE_MAN)
926 curp->pman = man_alloc(
927 curp->roff, curp, curp->defos,
928 curp->options & MPARSE_QUICK ? 1 : 0);
934 mparse_reset(struct mparse *curp)
937 roff_reset(curp->roff);
940 mdoc_reset(curp->mdoc);
942 man_reset(curp->man);
944 curp->secondary->sz = 0;
946 curp->file_status = MANDOCLEVEL_OK;
955 mparse_free(struct mparse *curp)
959 mdoc_free(curp->pmdoc);
961 man_free(curp->pman);
963 roff_free(curp->roff);
965 free(curp->secondary->buf);
967 free(curp->secondary);
973 mparse_result(struct mparse *curp,
974 struct mdoc **mdoc, struct man **man, char **sodest)
977 if (sodest && NULL != (*sodest = curp->sodest)) {
989 mandoc_vmsg(enum mandocerr t, struct mparse *m,
990 int ln, int pos, const char *fmt, ...)
996 (void)vsnprintf(buf, sizeof(buf), fmt, ap);
999 mandoc_msg(t, m, ln, pos, buf);
1003 mandoc_msg(enum mandocerr er, struct mparse *m,
1004 int ln, int col, const char *msg)
1006 enum mandoclevel level;
1008 level = MANDOCLEVEL_UNSUPP;
1009 while (er < mandoclimits[level])
1012 if (level < m->wlevel && er != MANDOCERR_FILE)
1016 (*m->mmsg)(er, level, m->file, ln, col, msg);
1018 if (m->file_status < level)
1019 m->file_status = level;
1023 mparse_strerror(enum mandocerr er)
1026 return(mandocerrs[er]);
1030 mparse_strlevel(enum mandoclevel lvl)
1032 return(mandoclevels[lvl]);
1036 mparse_keep(struct mparse *p)
1039 assert(NULL == p->secondary);
1040 p->secondary = mandoc_calloc(1, sizeof(struct buf));
1044 mparse_getkeep(const struct mparse *p)
1047 assert(p->secondary);
1048 return(p->secondary->sz ? p->secondary->buf : NULL);