contrib/one-true-awk/lex.c

   1 /****************************************************************
   2 Copyright (C) Lucent Technologies 1997
   3 All Rights Reserved
   4
   5 Permission to use, copy, modify, and distribute this software and
   6 its documentation for any purpose and without fee is hereby
   7 granted, provided that the above copyright notice appear in all
   8 copies and that both that the copyright notice and this
   9 permission notice and warranty disclaimer appear in supporting
  10 documentation, and that the name Lucent Technologies or any of
  11 its entities not be used in advertising or publicity pertaining
  12 to distribution of the software without specific, written prior
  13 permission.
  14
  15 LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
  16 INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
  17 IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY
  18 SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  19 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
  20 IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
  21 ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
  22 THIS SOFTWARE.
  23 ****************************************************************/
  24
  25 #include <stdio.h>
  26 #include <stdlib.h>
  27 #include <string.h>
  28 #include <ctype.h>
  29 #include "awk.h"
  30 #include "ytab.h"
  31
  32 extern YYSTYPE  yylval;
  33 extern int      infunc;
  34
  35 int     lineno  = 1;
  36 int     bracecnt = 0;
  37 int     brackcnt  = 0;
  38 int     parencnt = 0;
  39
  40 typedef struct Keyword {
  41         const char *word;
  42         int     sub;
  43         int     type;
  44 } Keyword;
  45
  46 Keyword keywords[] ={   /* keep sorted: binary searched */
  47         { "BEGIN",      XBEGIN,         XBEGIN },
  48         { "END",        XEND,           XEND },
  49         { "NF",         VARNF,          VARNF },
  50         { "and",        FAND,           BLTIN },
  51         { "atan2",      FATAN,          BLTIN },
  52         { "break",      BREAK,          BREAK },
  53         { "close",      CLOSE,          CLOSE },
  54         { "compl",      FCOMPL,         BLTIN },
  55         { "continue",   CONTINUE,       CONTINUE },
  56         { "cos",        FCOS,           BLTIN },
  57         { "delete",     DELETE,         DELETE },
  58         { "do",         DO,             DO },
  59         { "else",       ELSE,           ELSE },
  60         { "exit",       EXIT,           EXIT },
  61         { "exp",        FEXP,           BLTIN },
  62         { "fflush",     FFLUSH,         BLTIN },
  63         { "for",        FOR,            FOR },
  64         { "func",       FUNC,           FUNC },
  65         { "function",   FUNC,           FUNC },
  66         { "getline",    GETLINE,        GETLINE },
  67         { "gsub",       GSUB,           GSUB },
  68         { "if",         IF,             IF },
  69         { "in",         IN,             IN },
  70         { "index",      INDEX,          INDEX },
  71         { "int",        FINT,           BLTIN },
  72         { "length",     FLENGTH,        BLTIN },
  73         { "log",        FLOG,           BLTIN },
  74         { "lshift",     FLSHIFT,        BLTIN },
  75         { "match",      MATCHFCN,       MATCHFCN },
  76         { "next",       NEXT,           NEXT },
  77         { "nextfile",   NEXTFILE,       NEXTFILE },
  78         { "or",         FFOR,           BLTIN },
  79         { "print",      PRINT,          PRINT },
  80         { "printf",     PRINTF,         PRINTF },
  81         { "rand",       FRAND,          BLTIN },
  82         { "return",     RETURN,         RETURN },
  83         { "rshift",     FRSHIFT,        BLTIN },
  84         { "sin",        FSIN,           BLTIN },
  85         { "split",      SPLIT,          SPLIT },
  86         { "sprintf",    SPRINTF,        SPRINTF },
  87         { "sqrt",       FSQRT,          BLTIN },
  88         { "srand",      FSRAND,         BLTIN },
  89         { "sub",        SUB,            SUB },
  90         { "substr",     SUBSTR,         SUBSTR },
  91         { "system",     FSYSTEM,        BLTIN },
  92         { "tolower",    FTOLOWER,       BLTIN },
  93         { "toupper",    FTOUPPER,       BLTIN },
  94         { "while",      WHILE,          WHILE },
  95         { "xor",        FXOR,           BLTIN },
  96 };
  97
  98 #define RET(x)  { if(dbg)printf("lex %s\n", tokname(x)); return(x); }
  99
 100 int peek(void)
 101 {
 102         int c = input();
 103         unput(c);
 104         return c;
 105 }
 106
 107 int gettok(char **pbuf, int *psz)       /* get next input token */
 108 {
 109         int c, retc;
 110         char *buf = *pbuf;
 111         int sz = *psz;
 112         char *bp = buf;
 113
 114         c = input();
 115         if (c == 0)
 116                 return 0;
 117         buf[0] = c;
 118         buf[1] = 0;
 119         if (!isalnum(c) && c != '.' && c != '_')
 120                 return c;
 121
 122         *bp++ = c;
 123         if (isalpha(c) || c == '_') {   /* it's a varname */
 124                 for ( ; (c = input()) != 0; ) {
 125                         if (bp-buf >= sz)
 126                                 if (!adjbuf(&buf, &sz, bp-buf+2, 100, &bp, "gettok"))
 127                                         FATAL( "out of space for name %.10s...", buf );
 128                         if (isalnum(c) || c == '_')
 129                                 *bp++ = c;
 130                         else {
 131                                 *bp = 0;
 132                                 unput(c);
 133                                 break;
 134                         }
 135                 }
 136                 *bp = 0;
 137                 retc = 'a';     /* alphanumeric */
 138         } else {        /* maybe it's a number, but could be . */
 139                 char *rem;
 140                 /* read input until can't be a number */
 141                 for ( ; (c = input()) != 0; ) {
 142                         if (bp-buf >= sz)
 143                                 if (!adjbuf(&buf, &sz, bp-buf+2, 100, &bp, "gettok"))
 144                                         FATAL( "out of space for number %.10s...", buf );
 145                         if (isdigit(c) || c == 'e' || c == 'E'
 146                           || c == '.' || c == '+' || c == '-')
 147                                 *bp++ = c;
 148                         else {
 149                                 unput(c);
 150                                 break;
 151                         }
 152                 }
 153                 *bp = 0;
 154                 strtod(buf, &rem);      /* parse the number */
 155                 if (rem == buf) {       /* it wasn't a valid number at all */
 156                         buf[1] = 0;     /* return one character as token */
 157                         retc = buf[0];  /* character is its own type */
 158                         unputstr(rem+1); /* put rest back for later */
 159                 } else {        /* some prefix was a number */
 160                         unputstr(rem);  /* put rest back for later */
 161                         rem[0] = 0;     /* truncate buf after number part */
 162                         retc = '0';     /* type is number */
 163                 }
 164         }
 165         *pbuf = buf;
 166         *psz = sz;
 167         return retc;
 168 }
 169
 170 int     word(char *);
 171 int     string(void);
 172 int     regexpr(void);
 173 int     sc      = 0;    /* 1 => return a } right now */
 174 int     reg     = 0;    /* 1 => return a REGEXPR now */
 175
 176 int yylex(void)
 177 {
 178         int c;
 179         static char *buf = NULL;
 180         static int bufsize = 5; /* BUG: setting this small causes core dump! */
 181
 182         if (buf == NULL && (buf = (char *) malloc(bufsize)) == NULL)
 183                 FATAL( "out of space in yylex" );
 184         if (sc) {
 185                 sc = 0;
 186                 RET('}');
 187         }
 188         if (reg) {
 189                 reg = 0;
 190                 return regexpr();
 191         }
 192         for (;;) {
 193                 c = gettok(&buf, &bufsize);
 194                 if (c == 0)
 195                         return 0;
 196                 if (isalpha(c) || c == '_')
 197                         return word(buf);
 198                 if (isdigit(c)) {
 199                         yylval.cp = setsymtab(buf, tostring(buf), atof(buf), CON|NUM, symtab);
 200                         /* should this also have STR set? */
 201                         RET(NUMBER);
 202                 }
 203
 204                 yylval.i = c;
 205                 switch (c) {
 206                 case '\n':      /* {EOL} */
 207                         lineno++;
 208                         RET(NL);
 209                 case '\r':      /* assume \n is coming */
 210                 case ' ':       /* {WS}+ */
 211                 case '\t':
 212                         break;
 213                 case '#':       /* #.* strip comments */
 214                         while ((c = input()) != '\n' && c != 0)
 215                                 ;
 216                         unput(c);
 217                         break;
 218                 case ';':
 219                         RET(';');
 220                 case '\\':
 221                         if (peek() == '\n') {
 222                                 input();
 223                                 lineno++;
 224                         } else if (peek() == '\r') {
 225                                 input(); input();       /* \n */
 226                                 lineno++;
 227                         } else {
 228                                 RET(c);
 229                         }
 230                         break;
 231                 case '&':
 232                         if (peek() == '&') {
 233                                 input(); RET(AND);
 234                         } else
 235                                 RET('&');
 236                 case '|':
 237                         if (peek() == '|') {
 238                                 input(); RET(BOR);
 239                         } else
 240                                 RET('|');
 241                 case '!':
 242                         if (peek() == '=') {
 243                                 input(); yylval.i = NE; RET(NE);
 244                         } else if (peek() == '~') {
 245                                 input(); yylval.i = NOTMATCH; RET(MATCHOP);
 246                         } else
 247                                 RET(NOT);
 248                 case '~':
 249                         yylval.i = MATCH;
 250                         RET(MATCHOP);
 251                 case '<':
 252                         if (peek() == '=') {
 253                                 input(); yylval.i = LE; RET(LE);
 254                         } else {
 255                                 yylval.i = LT; RET(LT);
 256                         }
 257                 case '=':
 258                         if (peek() == '=') {
 259                                 input(); yylval.i = EQ; RET(EQ);
 260                         } else {
 261                                 yylval.i = ASSIGN; RET(ASGNOP);
 262                         }
 263                 case '>':
 264                         if (peek() == '=') {
 265                                 input(); yylval.i = GE; RET(GE);
 266                         } else if (peek() == '>') {
 267                                 input(); yylval.i = APPEND; RET(APPEND);
 268                         } else {
 269                                 yylval.i = GT; RET(GT);
 270                         }
 271                 case '+':
 272                         if (peek() == '+') {
 273                                 input(); yylval.i = INCR; RET(INCR);
 274                         } else if (peek() == '=') {
 275                                 input(); yylval.i = ADDEQ; RET(ASGNOP);
 276                         } else
 277                                 RET('+');
 278                 case '-':
 279                         if (peek() == '-') {
 280                                 input(); yylval.i = DECR; RET(DECR);
 281                         } else if (peek() == '=') {
 282                                 input(); yylval.i = SUBEQ; RET(ASGNOP);
 283                         } else
 284                                 RET('-');
 285                 case '*':
 286                         if (peek() == '=') {    /* *= */
 287                                 input(); yylval.i = MULTEQ; RET(ASGNOP);
 288                         } else if (peek() == '*') {     /* ** or **= */
 289                                 input();        /* eat 2nd * */
 290                                 if (peek() == '=') {
 291                                         input(); yylval.i = POWEQ; RET(ASGNOP);
 292                                 } else {
 293                                         RET(POWER);
 294                                 }
 295                         } else
 296                                 RET('*');
 297                 case '/':
 298                         RET('/');
 299                 case '%':
 300                         if (peek() == '=') {
 301                                 input(); yylval.i = MODEQ; RET(ASGNOP);
 302                         } else
 303                                 RET('%');
 304                 case '^':
 305                         if (peek() == '=') {
 306                                 input(); yylval.i = POWEQ; RET(ASGNOP);
 307                         } else
 308                                 RET(POWER);
 309
 310                 case '$':
 311                         /* BUG: awkward, if not wrong */
 312                         c = gettok(&buf, &bufsize);
 313                         if (isalpha(c)) {
 314                                 if (strcmp(buf, "NF") == 0) {   /* very special */
 315                                         unputstr("(NF)");
 316                                         RET(INDIRECT);
 317                                 }
 318                                 c = peek();
 319                                 if (c == '(' || c == '[' || (infunc && isarg(buf) >= 0)) {
 320                                         unputstr(buf);
 321                                         RET(INDIRECT);
 322                                 }
 323                                 yylval.cp = setsymtab(buf, "", 0.0, STR|NUM, symtab);
 324                                 RET(IVAR);
 325                         } else if (c == 0) {    /*  */
 326                                 SYNTAX( "unexpected end of input after $" );
 327                                 RET(';');
 328                         } else {
 329                                 unputstr(buf);
 330                                 RET(INDIRECT);
 331                         }
 332
 333                 case '}':
 334                         if (--bracecnt < 0)
 335                                 SYNTAX( "extra }" );
 336                         sc = 1;
 337                         RET(';');
 338                 case ']':
 339                         if (--brackcnt < 0)
 340                                 SYNTAX( "extra ]" );
 341                         RET(']');
 342                 case ')':
 343                         if (--parencnt < 0)
 344                                 SYNTAX( "extra )" );
 345                         RET(')');
 346                 case '{':
 347                         bracecnt++;
 348                         RET('{');
 349                 case '[':
 350                         brackcnt++;
 351                         RET('[');
 352                 case '(':
 353                         parencnt++;
 354                         RET('(');
 355
 356                 case '"':
 357                         return string();        /* BUG: should be like tran.c ? */
 358
 359                 default:
 360                         RET(c);
 361                 }
 362         }
 363 }
 364
 365 int string(void)
 366 {
 367         int c, n;
 368         char *s, *bp;
 369         static char *buf = NULL;
 370         static int bufsz = 500;
 371
 372         if (buf == NULL && (buf = (char *) malloc(bufsz)) == NULL)
 373                 FATAL("out of space for strings");
 374         for (bp = buf; (c = input()) != '"'; ) {
 375                 if (!adjbuf(&buf, &bufsz, bp-buf+2, 500, &bp, "string"))
 376                         FATAL("out of space for string %.10s...", buf);
 377                 switch (c) {
 378                 case '\n':
 379                 case '\r':
 380                 case 0:
 381                         *bp = '\0';
 382                         SYNTAX( "non-terminated string %.10s...", buf );
 383                         if (c == 0)     /* hopeless */
 384                                 FATAL( "giving up" );
 385                         lineno++;
 386                         break;
 387                 case '\\':
 388                         c = input();
 389                         switch (c) {
 390                         case '"': *bp++ = '"'; break;
 391                         case 'n': *bp++ = '\n'; break;
 392                         case 't': *bp++ = '\t'; break;
 393                         case 'f': *bp++ = '\f'; break;
 394                         case 'r': *bp++ = '\r'; break;
 395                         case 'b': *bp++ = '\b'; break;
 396                         case 'v': *bp++ = '\v'; break;
 397                         case 'a': *bp++ = '\007'; break;
 398                         case '\\': *bp++ = '\\'; break;
 399
 400                         case '0': case '1': case '2': /* octal: \d \dd \ddd */
 401                         case '3': case '4': case '5': case '6': case '7':
 402                                 n = c - '0';
 403                                 if ((c = peek()) >= '0' && c < '8') {
 404                                         n = 8 * n + input() - '0';
 405                                         if ((c = peek()) >= '0' && c < '8')
 406                                                 n = 8 * n + input() - '0';
 407                                 }
 408                                 *bp++ = n;
 409                                 break;
 410
 411                         case 'x':       /* hex  \x0-9a-fA-F + */
 412                             {   char xbuf[100], *px;
 413                                 for (px = xbuf; (c = input()) != 0 && px-xbuf < 100-2; ) {
 414                                         if (isdigit(c)
 415                                          || (c >= 'a' && c <= 'f')
 416                                          || (c >= 'A' && c <= 'F'))
 417                                                 *px++ = c;
 418                                         else
 419                                                 break;
 420                                 }
 421                                 *px = 0;
 422                                 unput(c);
 423                                 sscanf(xbuf, "%x", (unsigned int *) &n);
 424                                 *bp++ = n;
 425                                 break;
 426                             }
 427
 428                         default:
 429                                 *bp++ = c;
 430                                 break;
 431                         }
 432                         break;
 433                 default:
 434                         *bp++ = c;
 435                         break;
 436                 }
 437         }
 438         *bp = 0;
 439         s = tostring(buf);
 440         *bp++ = ' '; *bp++ = 0;
 441         yylval.cp = setsymtab(buf, s, 0.0, CON|STR|DONTFREE, symtab);
 442         RET(STRING);
 443 }
 444
 445
 446 int binsearch(char *w, Keyword *kp, int n)
 447 {
 448         int cond, low, mid, high;
 449
 450         low = 0;
 451         high = n - 1;
 452         while (low <= high) {
 453                 mid = (low + high) / 2;
 454                 if ((cond = strcmp(w, kp[mid].word)) < 0)
 455                         high = mid - 1;
 456                 else if (cond > 0)
 457                         low = mid + 1;
 458                 else
 459                         return mid;
 460         }
 461         return -1;
 462 }
 463
 464 int word(char *w)
 465 {
 466         Keyword *kp;
 467         int c, n;
 468
 469         n = binsearch(w, keywords, sizeof(keywords)/sizeof(keywords[0]));
 470         if (n != -1) {  /* found in table */
 471                 kp = keywords + n;
 472                 yylval.i = kp->sub;
 473                 switch (kp->type) {     /* special handling */
 474                 case BLTIN:
 475                         if (kp->sub == FSYSTEM && safe)
 476                                 SYNTAX( "system is unsafe" );
 477                         RET(kp->type);
 478                 case FUNC:
 479                         if (infunc)
 480                                 SYNTAX( "illegal nested function" );
 481                         RET(kp->type);
 482                 case RETURN:
 483                         if (!infunc)
 484                                 SYNTAX( "return not in function" );
 485                         RET(kp->type);
 486                 case VARNF:
 487                         yylval.cp = setsymtab("NF", "", 0.0, NUM, symtab);
 488                         RET(VARNF);
 489                 default:
 490                         RET(kp->type);
 491                 }
 492         }
 493         c = peek();     /* look for '(' */
 494         if (c != '(' && infunc && (n=isarg(w)) >= 0) {
 495                 yylval.i = n;
 496                 RET(ARG);
 497         } else {
 498                 yylval.cp = setsymtab(w, "", 0.0, STR|NUM|DONTFREE, symtab);
 499                 if (c == '(') {
 500                         RET(CALL);
 501                 } else {
 502                         RET(VAR);
 503                 }
 504         }
 505 }
 506
 507 void startreg(void)     /* next call to yylex will return a regular expression */
 508 {
 509         reg = 1;
 510 }
 511
 512 int regexpr(void)
 513 {
 514         int c;
 515         static char *buf = NULL;
 516         static int bufsz = 500;
 517         char *bp;
 518
 519         if (buf == NULL && (buf = (char *) malloc(bufsz)) == NULL)
 520                 FATAL("out of space for rex expr");
 521         bp = buf;
 522         for ( ; (c = input()) != '/' && c != 0; ) {
 523                 if (!adjbuf(&buf, &bufsz, bp-buf+3, 500, &bp, "regexpr"))
 524                         FATAL("out of space for reg expr %.10s...", buf);
 525                 if (c == '\n') {
 526                         *bp = '\0';
 527                         SYNTAX( "newline in regular expression %.10s...", buf );
 528                         unput('\n');
 529                         break;
 530                 } else if (c == '\\') {
 531                         *bp++ = '\\';
 532                         *bp++ = input();
 533                 } else {
 534                         *bp++ = c;
 535                 }
 536         }
 537         *bp = 0;
 538         if (c == 0)
 539                 SYNTAX("non-terminated regular expression %.10s...", buf);
 540         yylval.s = tostring(buf);
 541         unput('/');
 542         RET(REGEXPR);
 543 }
 544
 545 /* low-level lexical stuff, sort of inherited from lex */
 546
 547 char    ebuf[300];
 548 char    *ep = ebuf;
 549 char    yysbuf[100];    /* pushback buffer */
 550 char    *yysptr = yysbuf;
 551 FILE    *yyin = NULL;
 552
 553 int input(void) /* get next lexical input character */
 554 {
 555         int c;
 556         extern char *lexprog;
 557
 558         if (yysptr > yysbuf)
 559                 c = (uschar)*--yysptr;
 560         else if (lexprog != NULL) {     /* awk '...' */
 561                 if ((c = (uschar)*lexprog) != 0)
 562                         lexprog++;
 563         } else                          /* awk -f ... */
 564                 c = pgetc();
 565         if (c == EOF)
 566                 c = 0;
 567         if (ep >= ebuf + sizeof ebuf)
 568                 ep = ebuf;
 569         *ep = c;
 570         if (c != 0) {
 571                 ep++;
 572         }
 573         return (c);
 574 }
 575
 576 void unput(int c)       /* put lexical character back on input */
 577 {
 578         if (yysptr >= yysbuf + sizeof(yysbuf))
 579                 FATAL("pushed back too much: %.20s...", yysbuf);
 580         *yysptr++ = c;
 581         if (--ep < ebuf)
 582                 ep = ebuf + sizeof(ebuf) - 1;
 583 }
 584
 585 void unputstr(const char *s)    /* put a string back on input */
 586 {
 587         int i;
 588
 589         for (i = strlen(s)-1; i >= 0; i--)
 590                 unput(s[i]);
 591 }