2 * regcomp and regexec -- regsub and regerror are elsewhere
4 * Copyright (c) 1986 by University of Toronto.
5 * Written by Henry Spencer. Not derived from licensed software.
7 * Permission is granted to anyone to use this software for any
8 * purpose on any computer system, and to redistribute it freely,
9 * subject to the following restrictions:
11 * 1. The author is not responsible for the consequences of use of
12 * this software, no matter how awful, even if they arise
15 * 2. The origin of this software must not be misrepresented, either
16 * by explicit claim or by omission.
18 * 3. Altered versions must be plainly marked as such, and must not
19 * be misrepresented as being the original software.
20 *** THIS IS AN ALTERED VERSION. It was altered by John Gilmore,
21 *** hoptoad!gnu, on 27 Dec 1986, to add \n as an alternative to |
22 *** to assist in implementing egrep.
23 *** THIS IS AN ALTERED VERSION. It was altered by John Gilmore,
24 *** hoptoad!gnu, on 27 Dec 1986, to add \< and \> for word-matching
25 *** as in BSD grep and ex.
26 *** THIS IS AN ALTERED VERSION. It was altered by John Gilmore,
27 *** hoptoad!gnu, on 28 Dec 1986, to optimize characters quoted with \.
28 *** THIS IS AN ALTERED VERSION. It was altered by James A. Woods,
29 *** ames!jaw, on 19 June 1987, to quash a regcomp() redundancy.
31 * Beware that some of this code is subtly aware of the way operator
32 * precedence is structured in regular expressions. Serious changes in
33 * regular-expression syntax might require a total rethink.
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD$");
49 * The "internal use only" fields in regexp.h are present to pass info from
50 * compile to execute that permits the execute phase to run lots faster on
51 * simple cases. They are:
53 * regstart char that must begin a match; '\0' if none obvious
54 * reganch is the match anchored (at beginning-of-line only)?
55 * regmust string (pointer into program) that match must include, or NULL
56 * regmlen length of regmust string
58 * Regstart and reganch permit very fast decisions on suitable starting points
59 * for a match, cutting down the work a lot. Regmust permits fast rejection
60 * of lines that cannot possibly match. The regmust tests are costly enough
61 * that regcomp() supplies a regmust only if the r.e. contains something
62 * potentially expensive (at present, the only such thing detected is * or +
63 * at the start of the r.e., which can involve a lot of backup). Regmlen is
64 * supplied because the test in regexec() needs it and regcomp() is computing
69 * Structure for regexp "program". This is essentially a linear encoding
70 * of a nondeterministic finite-state machine (aka syntax charts or
71 * "railroad normal form" in parsing technology). Each node is an opcode
72 * plus a "next" pointer, possibly plus an operand. "Next" pointers of
73 * all nodes except BRANCH implement concatenation; a "next" pointer with
74 * a BRANCH on both ends of it is connecting two alternatives. (Here we
75 * have one of the subtle syntax dependencies: an individual BRANCH (as
76 * opposed to a collection of them) is never concatenated with anything
77 * because of operator precedence.) The operand of some types of node is
78 * a literal string; for others, it is a node leading into a sub-FSM. In
79 * particular, the operand of a BRANCH node is the first node of the branch.
80 * (NB this is *not* a tree structure: the tail of the branch connects
81 * to the thing following the set of BRANCHes.) The opcodes are:
84 /* definition number opnd? meaning */
85 #define END 0 /* no End of program. */
86 #define BOL 1 /* no Match "" at beginning of line. */
87 #define EOL 2 /* no Match "" at end of line. */
88 #define ANY 3 /* no Match any one character. */
89 #define ANYOF 4 /* str Match any character in this string. */
90 #define ANYBUT 5 /* str Match any character not in this string. */
91 #define BRANCH 6 /* node Match this alternative, or the next... */
92 #define BACK 7 /* no Match "", "next" ptr points backward. */
93 #define EXACTLY 8 /* str Match this string. */
94 #define NOTHING 9 /* no Match empty string. */
95 #define STAR 10 /* node Match this (simple) thing 0 or more times. */
96 #define PLUS 11 /* node Match this (simple) thing 1 or more times. */
97 #define WORDA 12 /* no Match "" at wordchar, where prev is nonword */
98 #define WORDZ 13 /* no Match "" at nonwordchar, where prev is word */
99 #define OPEN 20 /* no Mark this point in input as start of #n. */
100 /* OPEN+1 is number 1, etc. */
101 #define CLOSE 30 /* no Analogous to OPEN. */
106 * BRANCH The set of branches constituting a single choice are hooked
107 * together with their "next" pointers, since precedence prevents
108 * anything being concatenated to any individual branch. The
109 * "next" pointer of the last BRANCH in a choice points to the
110 * thing following the whole choice. This is also where the
111 * final "next" pointer of each individual branch points; each
112 * branch starts with the operand node of a BRANCH node.
114 * BACK Normal "next" pointers all implicitly point forward; BACK
115 * exists to make loop structures possible.
117 * STAR,PLUS '?', and complex '*' and '+', are implemented as circular
118 * BRANCH structures using BACK. Simple cases (one character
119 * per match) are implemented with STAR and PLUS for speed
120 * and to minimize recursive plunges.
122 * OPEN,CLOSE ...are numbered at compile time.
126 * A node is one char of opcode followed by two chars of "next" pointer.
127 * "Next" pointers are stored as two 8-bit pieces, high order first. The
128 * value is a positive offset from the opcode of the node containing it.
129 * An operand, if any, simply follows the node. (Note that much of the
130 * code generation knows about this implicit relationship.)
132 * Using two bytes for the "next" pointer is vast overkill for most things,
133 * but allows patterns to get big without disasters.
136 #define NEXT(p) (((*((p)+1)&0377)<<8) + (*((p)+2)&0377))
137 #define OPERAND(p) ((p) + 3)
140 * See regmagic.h for one further detail of program structure.
145 * Utility definitions.
148 #define UCHARAT(p) ((int)*(unsigned char *)(p))
150 #define UCHARAT(p) ((int)*(p)&CHARBITS)
153 #define FAIL(m) { regerror(m); return(NULL); }
154 #define ISMULT(c) ((c) == '*' || (c) == '+' || (c) == '?')
157 * Flags to be passed up and down.
159 #define HASWIDTH 01 /* Known never to match null string. */
160 #define SIMPLE 02 /* Simple enough to be STAR/PLUS operand. */
161 #define SPSTART 04 /* Starts with * or +. */
162 #define WORST 0 /* Worst case. */
165 * Global work variables for regcomp().
167 static char *regparse; /* Input-scan pointer. */
168 static int regnpar; /* () count. */
169 static char regdummy;
170 static char *regcode; /* Code-emit pointer; ®dummy = don't. */
171 static long regsize; /* Code size. */
174 * Forward declarations for regcomp()'s friends.
177 #define STATIC static
180 STATIC char *regbranch();
181 STATIC char *regpiece();
182 STATIC char *regatom();
183 STATIC char *regnode();
184 STATIC char *regnext();
186 STATIC void reginsert();
187 STATIC void regtail();
188 STATIC void regoptail();
190 STATIC int strcspn();
194 - regcomp - compile a regular expression into internal code
196 * We can't allocate space until we know how big the compiled form will be,
197 * but we can't compile it (and thus know how big it is) until we've got a
198 * place to put the code. So we cheat: we compile it twice, once with code
199 * generation turned off and size counting turned on, and once "for real".
200 * This also means that we don't allocate space until we are sure that the
201 * thing really will compile successfully, and we never have to move the
202 * code and thus invalidate pointers into it. (Note that it has to be in
203 * one piece because free() must be able to free it all.)
205 * Beware that the optimization-preparation code in here knows about some
206 * of the structure of the compiled regexp.
219 FAIL("NULL argument");
221 /* First pass: determine size, legality. */
223 if (exp[0] == '.' && exp[1] == '*') exp += 2; /* aid grep */
225 regparse = (char *)exp;
230 if (reg(0, &flags) == NULL)
233 /* Small enough for pointer-storage convention? */
234 if (regsize >= 32767L) /* Probably could be 65535L. */
235 FAIL("regexp too big");
237 /* Allocate space. */
238 r = (regexp *)malloc(sizeof(regexp) + (unsigned)regsize);
240 FAIL("out of space");
242 /* Second pass: emit code. */
243 regparse = (char *)exp;
245 regcode = r->program;
247 if (reg(0, &flags) == NULL)
250 /* Dig out information for optimizations. */
251 r->regstart = '\0'; /* Worst-case defaults. */
255 scan = r->program+1; /* First BRANCH. */
256 if (OP(regnext(scan)) == END) { /* Only one top-level choice. */
257 scan = OPERAND(scan);
259 /* Starting-point info. */
260 if (OP(scan) == EXACTLY)
261 r->regstart = *OPERAND(scan);
262 else if (OP(scan) == BOL)
266 * If there's something expensive in the r.e., find the
267 * longest literal string that must appear and make it the
268 * regmust. Resolve ties in favor of later strings, since
269 * the regstart check works with the beginning of the r.e.
270 * and avoiding duplication strengthens checking. Not a
271 * strong reason, but sufficient in the absence of others.
276 for (; scan != NULL; scan = regnext(scan))
277 if (OP(scan) == EXACTLY && strlen(OPERAND(scan)) >= len) {
278 longest = OPERAND(scan);
279 len = strlen(OPERAND(scan));
281 r->regmust = longest;
290 - reg - regular expression, i.e. main body or parenthesized thing
292 * Caller must absorb opening parenthesis.
294 * Combining parenthesis handling with the base level of regular expression
295 * is a trifle forced, but the need to tie the tails of the branches to what
296 * follows makes it hard to avoid.
300 int paren; /* Parenthesized? */
309 *flagp = HASWIDTH; /* Tentatively. */
311 /* Make an OPEN node, if parenthesized. */
313 if (regnpar >= NSUBEXP)
317 ret = regnode(OPEN+parno);
321 /* Pick up the branches, linking them together. */
322 br = regbranch(&flags);
326 regtail(ret, br); /* OPEN -> first. */
329 if (!(flags&HASWIDTH))
331 *flagp |= flags&SPSTART;
332 while (*regparse == '|' || *regparse == '\n') {
334 br = regbranch(&flags);
337 regtail(ret, br); /* BRANCH -> BRANCH. */
338 if (!(flags&HASWIDTH))
340 *flagp |= flags&SPSTART;
343 /* Make a closing node, and hook it on the end. */
344 ender = regnode((paren) ? CLOSE+parno : END);
347 /* Hook the tails of the branches to the closing node. */
348 for (br = ret; br != NULL; br = regnext(br))
349 regoptail(br, ender);
351 /* Check for proper termination. */
352 if (paren && *regparse++ != ')') {
353 FAIL("unmatched ()");
354 } else if (!paren && *regparse != '\0') {
355 if (*regparse == ')') {
356 FAIL("unmatched ()");
358 FAIL("junk on end"); /* "Can't happen". */
366 - regbranch - one alternative of an | operator
368 * Implements the concatenation operator.
379 *flagp = WORST; /* Tentatively. */
381 ret = regnode(BRANCH);
383 while (*regparse != '\0' && *regparse != ')' &&
384 *regparse != '\n' && *regparse != '|') {
385 latest = regpiece(&flags);
388 *flagp |= flags&HASWIDTH;
389 if (chain == NULL) /* First piece. */
390 *flagp |= flags&SPSTART;
392 regtail(chain, latest);
395 if (chain == NULL) /* Loop ran zero times. */
396 (void) regnode(NOTHING);
402 - regpiece - something followed by possible [*+?]
404 * Note that the branching code sequences used for ? and the general cases
405 * of * and + are somewhat optimized: they use the same NOTHING node as
406 * both the endmarker for their branch list and the body of the last branch.
407 * It might seem that this node could be dispensed with entirely, but the
408 * endmarker role is not redundant.
419 ret = regatom(&flags);
429 if (!(flags&HASWIDTH) && op != '?')
430 FAIL("*+ operand could be empty");
431 *flagp = (op != '+') ? (WORST|SPSTART) : (WORST|HASWIDTH);
433 if (op == '*' && (flags&SIMPLE))
434 reginsert(STAR, ret);
435 else if (op == '*') {
436 /* Emit x* as (x&|), where & means "self". */
437 reginsert(BRANCH, ret); /* Either x */
438 regoptail(ret, regnode(BACK)); /* and loop */
439 regoptail(ret, ret); /* back */
440 regtail(ret, regnode(BRANCH)); /* or */
441 regtail(ret, regnode(NOTHING)); /* null. */
442 } else if (op == '+' && (flags&SIMPLE))
443 reginsert(PLUS, ret);
444 else if (op == '+') {
445 /* Emit x+ as x(&|), where & means "self". */
446 next = regnode(BRANCH); /* Either */
448 regtail(regnode(BACK), ret); /* loop back */
449 regtail(next, regnode(BRANCH)); /* or */
450 regtail(ret, regnode(NOTHING)); /* null. */
451 } else if (op == '?') {
452 /* Emit x? as (x|) */
453 reginsert(BRANCH, ret); /* Either x */
454 regtail(ret, regnode(BRANCH)); /* or */
455 next = regnode(NOTHING); /* null. */
457 regoptail(ret, next);
460 if (ISMULT(*regparse))
467 - regatom - the lowest level
469 * Optimization: gobbles an entire sequence of ordinary characters so that
470 * it can turn them into a single node, which is smaller to store and
471 * faster to run. Backslashed characters are exceptions, each becoming a
472 * separate node; the code is simpler that way and it's not worth fixing.
481 *flagp = WORST; /* Tentatively. */
483 switch (*regparse++) {
484 /* FIXME: these chars only have meaning at beg/end of pat? */
493 *flagp |= HASWIDTH|SIMPLE;
500 if (*regparse == '^') { /* Complement of range. */
501 ret = regnode(ANYBUT);
504 ret = regnode(ANYOF);
505 if (*regparse == ']' || *regparse == '-')
507 while (*regparse != '\0' && *regparse != ']') {
508 if (*regparse == '-') {
510 if (*regparse == ']' || *regparse == '\0')
513 class = UCHARAT(regparse-2);
514 classend = UCHARAT(regparse);
515 if (__collate_load_error) {
516 if (class > classend)
517 FAIL("invalid [] range");
518 for (class++; class <= classend; class++)
521 if (__collate_range_cmp(class, classend) > 0)
522 FAIL("invalid [] range");
523 for (i = 0; i <= UCHAR_MAX; i++)
525 && __collate_range_cmp(class, i) <= 0
526 && __collate_range_cmp(i, classend) <= 0
536 if (*regparse != ']')
537 FAIL("unmatched []");
539 *flagp |= HASWIDTH|SIMPLE;
543 ret = reg(1, &flags);
546 *flagp |= flags&(HASWIDTH|SPSTART);
552 FAIL("internal urp"); /* Supposed to be caught earlier. */
557 FAIL("?+* follows nothing");
560 switch (*regparse++) {
565 ret = regnode(WORDA);
568 ret = regnode(WORDZ);
570 /* FIXME: Someday handle \1, \2, ... */
572 /* Handle general quoted chars in exact-match routine */
579 * Encode a string of characters to be matched exactly.
581 * This is a bit tricky due to quoted chars and due to
582 * '*', '+', and '?' taking the SINGLE char previous
585 * On entry, the char at regparse[-1] is going to go
586 * into the string, no matter what it is. (It could be
587 * following a \ if we are entered from the '\' case.)
589 * Basic idea is to pick up a good char in ch and
590 * examine the next char. If it's *+? then we twiddle.
591 * If it's \ then we frozzle. If it's other magic char
592 * we push ch and terminate the string. If none of the
593 * above, we push ch on the string and go around again.
595 * regprev is used to remember where "the current char"
596 * starts in the string, if due to a *+? we need to back
597 * up and put the current char in a separate, 1-char, string.
598 * When regprev is NULL, ch is the only char in the
599 * string; this is used in *+? handling, and in setting
600 * flags |= SIMPLE at the end.
606 regparse--; /* Look at cur char */
607 ret = regnode(EXACTLY);
608 for ( regprev = 0 ; ; ) {
609 ch = *regparse++; /* Get current char */
610 switch (*regparse) { /* look at next one */
613 regc(ch); /* Add cur to string */
616 case '.': case '[': case '(':
617 case ')': case '|': case '\n':
620 /* FIXME, $ and ^ should not always be magic */
622 regc(ch); /* dump cur char */
623 goto done; /* and we are done */
625 case '?': case '+': case '*':
626 if (!regprev) /* If just ch in str, */
627 goto magic; /* use it */
628 /* End mult-char string one early */
629 regparse = regprev; /* Back up parse */
633 regc(ch); /* Cur char OK */
634 switch (regparse[1]){ /* Look after \ */
638 /* FIXME: Someday handle \1, \2, ... */
639 goto done; /* Not quoted */
641 /* Backup point is \, scan * point is after it. */
644 continue; /* NOT break; */
647 regprev = regparse; /* Set backup point */
652 if (!regprev) /* One char? */
662 - regnode - emit a node
664 static char * /* Location. */
672 if (ret == ®dummy) {
679 *ptr++ = '\0'; /* Null "next" pointer. */
687 - regc - emit (if appropriate) a byte of code
693 if (regcode != ®dummy)
700 - reginsert - insert an operator in front of already-emitted operand
702 * Means relocating the operand.
713 if (regcode == ®dummy) {
724 place = opnd; /* Op node, where operand used to be. */
731 - regtail - set the next-pointer at the end of a node chain
745 /* Find last node. */
748 temp = regnext(scan);
754 if (OP(scan) == BACK)
758 *(scan+1) = (offset>>8)&0377;
759 *(scan+2) = offset&0377;
763 - regoptail - regtail on operand of first argument; nop if operandless
770 /* "Operandless" and "op != BRANCH" are synonymous in practice. */
771 if (p == NULL || p == ®dummy || OP(p) != BRANCH)
773 regtail(OPERAND(p), val);
777 * regexec and friends
781 * Global work variables for regexec().
783 static char *reginput; /* String-input pointer. */
784 static char *regbol; /* Beginning of input, for ^ check. */
785 static char **regstartp; /* Pointer to startp array. */
786 static char **regendp; /* Ditto for endp. */
792 STATIC int regmatch();
793 STATIC int regrepeat();
798 STATIC char *regprop();
802 - regexec - match a regexp against a string
805 regexec(prog, string)
810 extern char *strchr();
813 if (prog == NULL || string == NULL) {
814 regerror("NULL parameter");
818 /* Check validity of program. */
819 if (UCHARAT(prog->program) != MAGIC) {
820 regerror("corrupted program");
824 /* If there is a "must appear" string, look for it. */
825 if (prog->regmust != NULL) {
827 while ((s = strchr(s, prog->regmust[0])) != NULL) {
828 if (strncmp(s, prog->regmust, prog->regmlen) == 0)
829 break; /* Found it. */
832 if (s == NULL) /* Not present. */
836 /* Mark beginning of line for ^ . */
837 regbol = (char *)string;
839 /* Simplest case: anchored match need be tried only once. */
841 return(regtry(prog, string));
843 /* Messy cases: unanchored match. */
845 if (prog->regstart != '\0')
846 /* We know what char it must start with. */
847 while ((s = strchr(s, prog->regstart)) != NULL) {
853 /* We don't -- general case. */
857 } while (*s++ != '\0');
864 - regtry - try match at specific point
866 static int /* 0 failure, 1 success */
876 regstartp = prog->startp;
877 regendp = prog->endp;
881 for (i = NSUBEXP; i > 0; i--) {
885 if (regmatch(prog->program + 1)) {
886 prog->startp[0] = string;
887 prog->endp[0] = reginput;
894 - regmatch - main matching routine
896 * Conceptually the strategy is simple: check to see whether the current
897 * node matches, call self recursively to see whether the rest matches,
898 * and then act accordingly. In practice we make some effort to avoid
899 * recursion, in particular by going through "ordinary" nodes (that don't
900 * need to know whether the rest of the match failed) by a loop instead of
903 static int /* 0 failure, 1 success */
907 char *scan; /* Current node. */
908 char *next; /* Next node. */
912 if (scan != NULL && regnarrate)
913 fprintf(stderr, "%s(\n", regprop(scan));
915 while (scan != NULL) {
918 fprintf(stderr, "%s...\n", regprop(scan));
920 next = regnext(scan);
924 if (reginput != regbol)
928 if (*reginput != '\0')
932 /* Must be looking at a letter, digit, or _ */
933 if ((!isalnum((unsigned char)*reginput)) && *reginput != '_')
935 /* Prev must be BOL or nonword */
936 if (reginput > regbol &&
937 (isalnum((unsigned char)reginput[-1]) || reginput[-1] == '_'))
941 /* Must be looking at non letter, digit, or _ */
942 if (isalnum((unsigned char)*reginput) || *reginput == '_')
944 /* We don't care what the previous char was */
947 if (*reginput == '\0')
955 opnd = OPERAND(scan);
956 /* Inline the first character, for speed. */
957 if (*opnd != *reginput)
960 if (len > 1 && strncmp(opnd, reginput, len) != 0)
966 if (*reginput == '\0' || strchr(OPERAND(scan), *reginput) == NULL)
971 if (*reginput == '\0' || strchr(OPERAND(scan), *reginput) != NULL)
991 no = OP(scan) - OPEN;
994 if (regmatch(next)) {
996 * Don't set startp if some later
997 * invocation of the same parentheses
1000 if (regstartp[no] == NULL)
1001 regstartp[no] = save;
1019 no = OP(scan) - CLOSE;
1022 if (regmatch(next)) {
1024 * Don't set endp if some later
1025 * invocation of the same parentheses
1028 if (regendp[no] == NULL)
1038 if (OP(next) != BRANCH) /* No choice. */
1039 next = OPERAND(scan); /* Avoid recursion. */
1043 if (regmatch(OPERAND(scan)))
1046 scan = regnext(scan);
1047 } while (scan != NULL && OP(scan) == BRANCH);
1061 * Lookahead to avoid useless match attempts
1062 * when we know what character comes next.
1065 if (OP(next) == EXACTLY)
1066 nextch = *OPERAND(next);
1067 min = (OP(scan) == STAR) ? 0 : 1;
1069 no = regrepeat(OPERAND(scan));
1071 /* If it could work, try it. */
1072 if (nextch == '\0' || *reginput == nextch)
1075 /* Couldn't or didn't -- back up. */
1077 reginput = save + no;
1083 return(1); /* Success! */
1086 regerror("memory corruption");
1095 * We get here only if there's trouble -- normally "case END" is
1096 * the terminating point.
1098 regerror("corrupted pointers");
1103 - regrepeat - repeatedly match something simple, report how many
1117 count = strlen(scan);
1121 while (*opnd == *scan) {
1127 while (*scan != '\0' && strchr(opnd, *scan) != NULL) {
1133 while (*scan != '\0' && strchr(opnd, *scan) == NULL) {
1138 default: /* Oh dear. Called inappropriately. */
1139 regerror("internal foulup");
1140 count = 0; /* Best compromise. */
1149 - regnext - dig the "next" pointer out of a node
1172 STATIC char *regprop();
1175 - regdump - dump a regexp onto stdout in vaguely comprehensible form
1182 char op = EXACTLY; /* Arbitrary non-END op. */
1184 extern char *strchr();
1188 while (op != END) { /* While that wasn't END last time... */
1190 printf("%2d%s", s-r->program, regprop(s)); /* Where, what. */
1192 if (next == NULL) /* Next ptr. */
1195 printf("(%d)", (s-r->program)+(next-s));
1197 if (op == ANYOF || op == ANYBUT || op == EXACTLY) {
1198 /* Literal string, where present. */
1199 while (*s != '\0') {
1208 /* Header fields of interest. */
1209 if (r->regstart != '\0')
1210 printf("start `%c' ", r->regstart);
1212 printf("anchored ");
1213 if (r->regmust != NULL)
1214 printf("must have \"%s\"", r->regmust);
1219 - regprop - printable representation of opcode
1226 static char buf[50];
1228 (void) strcpy(buf, ":");
1270 sprintf(buf+strlen(buf), "OPEN%d", OP(op)-OPEN);
1282 sprintf(buf+strlen(buf), "CLOSE%d", OP(op)-CLOSE);
1298 regerror("corrupted opcode");
1302 (void) strcat(buf, p);
1308 * The following is provided for those people who do not have strcspn() in
1309 * their C libraries. They should get off their butts and do something
1310 * about it; at least one public-domain implementation of those (highly
1311 * useful) string routines has been published on Usenet.
1315 * strcspn - find length of initial segment of s1 consisting entirely
1316 * of characters not from s2
1329 for (scan1 = s1; *scan1 != '\0'; scan1++) {
1330 for (scan2 = s2; *scan2 != '\0';) /* ++ moved down. */
1331 if (*scan1 == *scan2++)