2 * regcomp and regexec -- regsub and regerror are elsewhere
4 * Copyright (c) 1986 by University of Toronto.
5 * Written by Henry Spencer. Not derived from licensed software.
7 * Permission is granted to anyone to use this software for any
8 * purpose on any computer system, and to redistribute it freely,
9 * subject to the following restrictions:
11 * 1. The author is not responsible for the consequences of use of
12 * this software, no matter how awful, even if they arise
15 * 2. The origin of this software must not be misrepresented, either
16 * by explicit claim or by omission.
18 * 3. Altered versions must be plainly marked as such, and must not
19 * be misrepresented as being the original software.
21 * Beware that some of this code is subtly aware of the way operator
22 * precedence is structured in regular expressions. Serious changes in
23 * regular-expression syntax might require a total rethink.
25 * *** NOTE: this code has been altered slightly for use in Tcl. ***
26 * Slightly modified by David MacKenzie to undo most of the changes for TCL.
27 * Added regexec2 with notbol parameter. -- 4/19/99 Mark Nudelman
43 * The "internal use only" fields in regexp.h are present to pass info from
44 * compile to execute that permits the execute phase to run lots faster on
45 * simple cases. They are:
47 * regstart char that must begin a match; '\0' if none obvious
48 * reganch is the match anchored (at beginning-of-line only)?
49 * regmust string (pointer into program) that match must include, or NULL
50 * regmlen length of regmust string
52 * Regstart and reganch permit very fast decisions on suitable starting points
53 * for a match, cutting down the work a lot. Regmust permits fast rejection
54 * of lines that cannot possibly match. The regmust tests are costly enough
55 * that regcomp() supplies a regmust only if the r.e. contains something
56 * potentially expensive (at present, the only such thing detected is * or +
57 * at the start of the r.e., which can involve a lot of backup). Regmlen is
58 * supplied because the test in regexec() needs it and regcomp() is
59 * computing it anyway.
63 * Structure for regexp "program". This is essentially a linear encoding
64 * of a nondeterministic finite-state machine (aka syntax charts or
65 * "railroad normal form" in parsing technology). Each node is an opcode
66 * plus a "next" pointer, possibly plus an operand. "Next" pointers of
67 * all nodes except BRANCH implement concatenation; a "next" pointer with
68 * a BRANCH on both ends of it is connecting two alternatives. (Here we
69 * have one of the subtle syntax dependencies: an individual BRANCH (as
70 * opposed to a collection of them) is never concatenated with anything
71 * because of operator precedence.) The operand of some types of node is
72 * a literal string; for others, it is a node leading into a sub-FSM. In
73 * particular, the operand of a BRANCH node is the first node of the branch.
74 * (NB this is *not* a tree structure: the tail of the branch connects
75 * to the thing following the set of BRANCHes.) The opcodes are:
78 /* definition number opnd? meaning */
80 #define END 0 /* no End of program. */
81 #define BOL 1 /* no Match "" at beginning of line. */
82 #define EOL 2 /* no Match "" at end of line. */
83 #define ANY 3 /* no Match any one character. */
84 #define ANYOF 4 /* str Match any character in this string. */
85 #define ANYBUT 5 /* str Match any character not in this string. */
86 #define BRANCH 6 /* node Match this alternative, or the next... */
87 #define BACK 7 /* no Match "", "next" ptr points backward. */
88 #define EXACTLY 8 /* str Match this string. */
89 #define NOTHING 9 /* no Match empty string. */
90 #define STAR 10 /* node Match this (simple) thing 0 or more times. */
91 #define PLUS 11 /* node Match this (simple) thing 1 or more times. */
92 #define OPEN 20 /* no Mark this point in input as start of #n. */
93 /* OPEN+1 is number 1, etc. */
94 #define CLOSE 30 /* no Analogous to OPEN. */
99 * BRANCH The set of branches constituting a single choice are hooked
100 * together with their "next" pointers, since precedence prevents
101 * anything being concatenated to any individual branch. The
102 * "next" pointer of the last BRANCH in a choice points to the
103 * thing following the whole choice. This is also where the
104 * final "next" pointer of each individual branch points; each
105 * branch starts with the operand node of a BRANCH node.
107 * BACK Normal "next" pointers all implicitly point forward; BACK
108 * exists to make loop structures possible.
110 * STAR,PLUS '?', and complex '*' and '+', are implemented as circular
111 * BRANCH structures using BACK. Simple cases (one character
112 * per match) are implemented with STAR and PLUS for speed
113 * and to minimize recursive plunges.
115 * OPEN,CLOSE ...are numbered at compile time.
119 * A node is one char of opcode followed by two chars of "next" pointer.
120 * "Next" pointers are stored as two 8-bit pieces, high order first. The
121 * value is a positive offset from the opcode of the node containing it.
122 * An operand, if any, simply follows the node. (Note that much of the
123 * code generation knows about this implicit relationship.)
125 * Using two bytes for the "next" pointer is vast overkill for most things,
126 * but allows patterns to get big without disasters.
129 #define NEXT(p) (((*((p)+1)&0377)<<8) + (*((p)+2)&0377))
130 #define OPERAND(p) ((p) + 3)
133 * See regmagic.h for one further detail of program structure.
138 * Utility definitions.
141 #define UCHARAT(p) ((int)*(unsigned char *)(p))
143 #define UCHARAT(p) ((int)*(p)&CHARBITS)
146 #define FAIL(m) { regerror(m); return(NULL); }
147 #define ISMULT(c) ((c) == '*' || (c) == '+' || (c) == '?')
148 #define META "^$.[()|?+*\\"
151 * Flags to be passed up and down.
153 #define HASWIDTH 01 /* Known never to match null string. */
154 #define SIMPLE 02 /* Simple enough to be STAR/PLUS operand. */
155 #define SPSTART 04 /* Starts with * or +. */
156 #define WORST 0 /* Worst case. */
159 * Global work variables for regcomp().
161 static char *regparse; /* Input-scan pointer. */
162 static int regnpar; /* () count. */
163 static char regdummy;
164 static char *regcode; /* Code-emit pointer; ®dummy = don't. */
165 static long regsize; /* Code size. */
168 * The first byte of the regexp internal "program" is actually this magic
169 * number; the start node begins in the second byte.
175 * Forward declarations for regcomp()'s friends.
178 #define STATIC static
181 STATIC char *regbranch();
182 STATIC char *regpiece();
183 STATIC char *regatom();
184 STATIC char *regnode();
185 STATIC char *regnext();
187 STATIC void reginsert();
188 STATIC void regtail();
189 STATIC void regoptail();
191 STATIC int strcspn();
195 - regcomp - compile a regular expression into internal code
197 * We can't allocate space until we know how big the compiled form will be,
198 * but we can't compile it (and thus know how big it is) until we've got a
199 * place to put the code. So we cheat: we compile it twice, once with code
200 * generation turned off and size counting turned on, and once "for real".
201 * This also means that we don't allocate space until we are sure that the
202 * thing really will compile successfully, and we never have to move the
203 * code and thus invalidate pointers into it. (Note that it has to be in
204 * one piece because free() must be able to free it all.)
206 * Beware that the optimization-preparation code in here knows about some
207 * of the structure of the compiled regexp.
215 register char *longest;
220 FAIL("NULL argument");
222 /* First pass: determine size, legality. */
228 if (reg(0, &flags) == NULL)
231 /* Small enough for pointer-storage convention? */
232 if (regsize >= 32767L) /* Probably could be 65535L. */
233 FAIL("regexp too big");
235 /* Allocate space. */
236 r = (regexp *)malloc(sizeof(regexp) + (unsigned)regsize);
238 FAIL("out of space");
240 /* Second pass: emit code. */
243 regcode = r->program;
245 if (reg(0, &flags) == NULL)
251 /* Dig out information for optimizations. */
252 r->regstart = '\0'; /* Worst-case defaults. */
256 scan = r->program+1; /* First BRANCH. */
257 if (OP(regnext(scan)) == END) { /* Only one top-level choice. */
258 scan = OPERAND(scan);
260 /* Starting-point info. */
261 if (OP(scan) == EXACTLY)
262 r->regstart = *OPERAND(scan);
263 else if (OP(scan) == BOL)
267 * If there's something expensive in the r.e., find the
268 * longest literal string that must appear and make it the
269 * regmust. Resolve ties in favor of later strings, since
270 * the regstart check works with the beginning of the r.e.
271 * and avoiding duplication strengthens checking. Not a
272 * strong reason, but sufficient in the absence of others.
277 for (; scan != NULL; scan = regnext(scan))
278 if (OP(scan) == EXACTLY && ((int) strlen(OPERAND(scan))) >= len) {
279 longest = OPERAND(scan);
280 len = (int) strlen(OPERAND(scan));
282 r->regmust = longest;
291 - reg - regular expression, i.e. main body or parenthesized thing
293 * Caller must absorb opening parenthesis.
295 * Combining parenthesis handling with the base level of regular expression
296 * is a trifle forced, but the need to tie the tails of the branches to what
297 * follows makes it hard to avoid.
301 int paren; /* Parenthesized? */
306 register char *ender;
307 register int parno = 0;
310 *flagp = HASWIDTH; /* Tentatively. */
312 /* Make an OPEN node, if parenthesized. */
314 if (regnpar >= NSUBEXP)
318 ret = regnode(OPEN+parno);
322 /* Pick up the branches, linking them together. */
323 br = regbranch(&flags);
327 regtail(ret, br); /* OPEN -> first. */
330 if (!(flags&HASWIDTH))
332 *flagp |= flags&SPSTART;
333 while (*regparse == '|') {
335 br = regbranch(&flags);
338 regtail(ret, br); /* BRANCH -> BRANCH. */
339 if (!(flags&HASWIDTH))
341 *flagp |= flags&SPSTART;
344 /* Make a closing node, and hook it on the end. */
345 ender = regnode((paren) ? CLOSE+parno : END);
348 /* Hook the tails of the branches to the closing node. */
349 for (br = ret; br != NULL; br = regnext(br))
350 regoptail(br, ender);
352 /* Check for proper termination. */
353 if (paren && *regparse++ != ')') {
354 FAIL("unmatched ()");
355 } else if (!paren && *regparse != '\0') {
356 if (*regparse == ')') {
357 FAIL("unmatched ()");
359 FAIL("junk on end"); /* "Can't happen". */
367 - regbranch - one alternative of an | operator
369 * Implements the concatenation operator.
376 register char *chain;
377 register char *latest;
380 *flagp = WORST; /* Tentatively. */
382 ret = regnode(BRANCH);
384 while (*regparse != '\0' && *regparse != '|' && *regparse != ')') {
385 latest = regpiece(&flags);
388 *flagp |= flags&HASWIDTH;
389 if (chain == NULL) /* First piece. */
390 *flagp |= flags&SPSTART;
392 regtail(chain, latest);
395 if (chain == NULL) /* Loop ran zero times. */
396 (void) regnode(NOTHING);
402 - regpiece - something followed by possible [*+?]
404 * Note that the branching code sequences used for ? and the general cases
405 * of * and + are somewhat optimized: they use the same NOTHING node as
406 * both the endmarker for their branch list and the body of the last branch.
407 * It might seem that this node could be dispensed with entirely, but the
408 * endmarker role is not redundant.
419 ret = regatom(&flags);
429 if (!(flags&HASWIDTH) && op != '?')
430 FAIL("*+ operand could be empty");
431 *flagp = (op != '+') ? (WORST|SPSTART) : (WORST|HASWIDTH);
433 if (op == '*' && (flags&SIMPLE))
434 reginsert(STAR, ret);
435 else if (op == '*') {
436 /* Emit x* as (x&|), where & means "self". */
437 reginsert(BRANCH, ret); /* Either x */
438 regoptail(ret, regnode(BACK)); /* and loop */
439 regoptail(ret, ret); /* back */
440 regtail(ret, regnode(BRANCH)); /* or */
441 regtail(ret, regnode(NOTHING)); /* null. */
442 } else if (op == '+' && (flags&SIMPLE))
443 reginsert(PLUS, ret);
444 else if (op == '+') {
445 /* Emit x+ as x(&|), where & means "self". */
446 next = regnode(BRANCH); /* Either */
448 regtail(regnode(BACK), ret); /* loop back */
449 regtail(next, regnode(BRANCH)); /* or */
450 regtail(ret, regnode(NOTHING)); /* null. */
451 } else if (op == '?') {
452 /* Emit x? as (x|) */
453 reginsert(BRANCH, ret); /* Either x */
454 regtail(ret, regnode(BRANCH)); /* or */
455 next = regnode(NOTHING); /* null. */
457 regoptail(ret, next);
460 if (ISMULT(*regparse))
467 - regatom - the lowest level
469 * Optimization: gobbles an entire sequence of ordinary characters so that
470 * it can turn them into a single node, which is smaller to store and
471 * faster to run. Backslashed characters are exceptions, each becoming a
472 * separate node; the code is simpler that way and it's not worth fixing.
481 *flagp = WORST; /* Tentatively. */
483 switch (*regparse++) {
492 *flagp |= HASWIDTH|SIMPLE;
496 register int classend;
498 if (*regparse == '^') { /* Complement of range. */
499 ret = regnode(ANYBUT);
502 ret = regnode(ANYOF);
503 if (*regparse == ']' || *regparse == '-')
505 while (*regparse != '\0' && *regparse != ']') {
506 if (*regparse == '-') {
508 if (*regparse == ']' || *regparse == '\0')
511 clss = UCHARAT(regparse-2)+1;
512 classend = UCHARAT(regparse);
513 if (clss > classend+1)
514 FAIL("invalid [] range");
515 for (; clss <= classend; clss++)
523 if (*regparse != ']')
524 FAIL("unmatched []");
526 *flagp |= HASWIDTH|SIMPLE;
530 ret = reg(1, &flags);
533 *flagp |= flags&(HASWIDTH|SPSTART);
538 FAIL("internal urp"); /* Supposed to be caught earlier. */
544 FAIL("?+* follows nothing");
548 if (*regparse == '\0')
550 ret = regnode(EXACTLY);
553 *flagp |= HASWIDTH|SIMPLE;
560 len = (int) strcspn(regparse, META);
562 FAIL("internal disaster");
563 ender = *(regparse+len);
564 if (len > 1 && ISMULT(ender))
565 len--; /* Back off clear of ?+* operand. */
569 ret = regnode(EXACTLY);
583 - regnode - emit a node
585 static char * /* Location. */
593 if (ret == ®dummy) {
600 *ptr++ = '\0'; /* Null "next" pointer. */
608 - regc - emit (if appropriate) a byte of code
614 if (regcode != ®dummy)
621 - reginsert - insert an operator in front of already-emitted operand
623 * Means relocating the operand.
632 register char *place;
634 if (regcode == ®dummy) {
645 place = opnd; /* Op node, where operand used to be. */
652 - regtail - set the next-pointer at the end of a node chain
666 /* Find last node. */
669 temp = regnext(scan);
675 if (OP(scan) == BACK)
676 offset = (int) (scan - val);
678 offset = (int) (val - scan);
679 *(scan+1) = (offset>>8)&0377;
680 *(scan+2) = offset&0377;
684 - regoptail - regtail on operand of first argument; nop if operandless
691 /* "Operandless" and "op != BRANCH" are synonymous in practice. */
692 if (p == NULL || p == ®dummy || OP(p) != BRANCH)
694 regtail(OPERAND(p), val);
698 * regexec and friends
702 * Global work variables for regexec().
704 static char *reginput; /* String-input pointer. */
705 static char *regbol; /* Beginning of input, for ^ check. */
706 static char **regstartp; /* Pointer to startp array. */
707 static char **regendp; /* Ditto for endp. */
713 STATIC int regmatch();
714 STATIC int regrepeat();
719 STATIC char *regprop();
723 - regexec - match a regexp against a string
726 regexec2(prog, string, notbol)
727 register regexp *prog;
728 register char *string;
734 if (prog == NULL || string == NULL) {
735 regerror("NULL parameter");
739 /* Check validity of program. */
740 if (UCHARAT(prog->program) != MAGIC) {
741 regerror("corrupted program");
745 /* If there is a "must appear" string, look for it. */
746 if (prog->regmust != NULL) {
748 while ((s = strchr(s, prog->regmust[0])) != NULL) {
749 if (strncmp(s, prog->regmust, prog->regmlen) == 0)
750 break; /* Found it. */
753 if (s == NULL) /* Not present. */
757 /* Mark beginning of line for ^ . */
763 /* Simplest case: anchored match need be tried only once. */
765 return(regtry(prog, string));
767 /* Messy cases: unanchored match. */
769 if (prog->regstart != '\0')
770 /* We know what char it must start with. */
771 while ((s = strchr(s, prog->regstart)) != NULL) {
777 /* We don't -- general case. */
781 } while (*s++ != '\0');
788 regexec(prog, string)
789 register regexp *prog;
790 register char *string;
792 return regexec2(prog, string, 0);
796 - regtry - try match at specific point
798 static int /* 0 failure, 1 success */
808 regstartp = prog->startp;
809 regendp = prog->endp;
813 for (i = NSUBEXP; i > 0; i--) {
817 if (regmatch(prog->program + 1)) {
818 prog->startp[0] = string;
819 prog->endp[0] = reginput;
826 - regmatch - main matching routine
828 * Conceptually the strategy is simple: check to see whether the current
829 * node matches, call self recursively to see whether the rest matches,
830 * and then act accordingly. In practice we make some effort to avoid
831 * recursion, in particular by going through "ordinary" nodes (that don't
832 * need to know whether the rest of the match failed) by a loop instead of
835 static int /* 0 failure, 1 success */
839 register char *scan; /* Current node. */
840 char *next; /* Next node. */
844 if (scan != NULL && regnarrate)
845 fprintf(stderr, "%s(\n", regprop(scan));
847 while (scan != NULL) {
850 fprintf(stderr, "%s...\n", regprop(scan));
852 next = regnext(scan);
856 if (reginput != regbol)
860 if (*reginput != '\0')
864 if (*reginput == '\0')
872 opnd = OPERAND(scan);
873 /* Inline the first character, for speed. */
874 if (*opnd != *reginput)
876 len = (int) strlen(opnd);
877 if (len > 1 && strncmp(opnd, reginput, len) != 0)
883 if (*reginput == '\0' || strchr(OPERAND(scan), *reginput) == NULL)
888 if (*reginput == '\0' || strchr(OPERAND(scan), *reginput) != NULL)
908 no = OP(scan) - OPEN;
911 if (regmatch(next)) {
913 * Don't set startp if some later
914 * invocation of the same parentheses
917 if (regstartp[no] == NULL)
918 regstartp[no] = save;
937 no = OP(scan) - CLOSE;
940 if (regmatch(next)) {
942 * Don't set endp if some later
943 * invocation of the same parentheses
946 if (regendp[no] == NULL)
957 if (OP(next) != BRANCH) /* No choice. */
958 next = OPERAND(scan); /* Avoid recursion. */
962 if (regmatch(OPERAND(scan)))
965 scan = regnext(scan);
966 } while (scan != NULL && OP(scan) == BRANCH);
975 register char nextch;
981 * Lookahead to avoid useless match attempts
982 * when we know what character comes next.
985 if (OP(next) == EXACTLY)
986 nextch = *OPERAND(next);
987 min = (OP(scan) == STAR) ? 0 : 1;
989 no = regrepeat(OPERAND(scan));
991 /* If it could work, try it. */
992 if (nextch == '\0' || *reginput == nextch)
995 /* Couldn't or didn't -- back up. */
997 reginput = save + no;
1004 return(1); /* Success! */
1008 regerror("memory corruption");
1018 * We get here only if there's trouble -- normally "case END" is
1019 * the terminating point.
1021 regerror("corrupted pointers");
1026 - regrepeat - repeatedly match something simple, report how many
1032 register int count = 0;
1033 register char *scan;
1034 register char *opnd;
1040 count = (int) strlen(scan);
1044 while (*opnd == *scan) {
1050 while (*scan != '\0' && strchr(opnd, *scan) != NULL) {
1056 while (*scan != '\0' && strchr(opnd, *scan) == NULL) {
1061 default: /* Oh dear. Called inappropriately. */
1062 regerror("internal foulup");
1063 count = 0; /* Best compromise. */
1072 - regnext - dig the "next" pointer out of a node
1078 register int offset;
1095 STATIC char *regprop();
1098 - regdump - dump a regexp onto stdout in vaguely comprehensible form
1105 register char op = EXACTLY; /* Arbitrary non-END op. */
1106 register char *next;
1110 while (op != END) { /* While that wasn't END last time... */
1112 printf("%2d%s", s-r->program, regprop(s)); /* Where, what. */
1114 if (next == NULL) /* Next ptr. */
1117 printf("(%d)", (s-r->program)+(next-s));
1119 if (op == ANYOF || op == ANYBUT || op == EXACTLY) {
1120 /* Literal string, where present. */
1121 while (*s != '\0') {
1130 /* Header fields of interest. */
1131 if (r->regstart != '\0')
1132 printf("start `%c' ", r->regstart);
1134 printf("anchored ");
1135 if (r->regmust != NULL)
1136 printf("must have \"%s\"", r->regmust);
1141 - regprop - printable representation of opcode
1148 static char buf[50];
1150 (void) strcpy(buf, ":");
1192 sprintf(buf+strlen(buf), "OPEN%d", OP(op)-OPEN);
1204 sprintf(buf+strlen(buf), "CLOSE%d", OP(op)-CLOSE);
1214 regerror("corrupted opcode");
1218 (void) strcat(buf, p);
1224 * The following is provided for those people who do not have strcspn() in
1225 * their C libraries. They should get off their butts and do something
1226 * about it; at least one public-domain implementation of those (highly
1227 * useful) string routines has been published on Usenet.
1231 * strcspn - find length of initial segment of s1 consisting entirely
1232 * of characters not from s2
1240 register char *scan1;
1241 register char *scan2;
1245 for (scan1 = s1; *scan1 != '\0'; scan1++) {
1246 for (scan2 = s2; *scan2 != '\0';) /* ++ moved down. */
1247 if (*scan1 == *scan2++)