2 * Copyright (C) 2004, 2005, 2007, 2008 Internet Systems Consortium, Inc. ("ISC")
3 * Copyright (C) 1998-2002 Internet Software Consortium.
5 * Permission to use, copy, modify, and/or distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
9 * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
10 * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
11 * AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
12 * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
13 * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
14 * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
15 * PERFORMANCE OF THIS SOFTWARE.
18 /* $Id: lex.h,v 1.37 2008/05/30 23:47:01 tbox Exp $ */
28 * \brief The "lex" module provides a lightweight tokenizer. It can operate
29 * on files or buffers, and can handle "include". It is designed for
30 * parsing of DNS master files and the BIND configuration file, but
31 * should be general enough to tokenize other things, e.g. HTTP.
34 * No synchronization is provided. Clients must ensure exclusive
38 * No anticipated impact.
44 * No anticipated impact.
57 #include <isc/region.h>
58 #include <isc/types.h>
68 * Various options for isc_lex_gettoken().
71 #define ISC_LEXOPT_EOL 0x01 /*%< Want end-of-line token. */
72 #define ISC_LEXOPT_EOF 0x02 /*%< Want end-of-file token. */
73 #define ISC_LEXOPT_INITIALWS 0x04 /*%< Want initial whitespace. */
74 #define ISC_LEXOPT_NUMBER 0x08 /*%< Recognize numbers. */
75 #define ISC_LEXOPT_QSTRING 0x10 /*%< Recognize qstrings. */
80 * The ISC_LEXOPT_DNSMULTILINE option handles the processing of '(' and ')' in
81 * the DNS master file format. If this option is set, then the
82 * ISC_LEXOPT_INITIALWS and ISC_LEXOPT_EOL options will be ignored when
83 * the paren count is > 0. To use this option, '(' and ')' must be special
86 #define ISC_LEXOPT_DNSMULTILINE 0x20 /*%< Handle '(' and ')'. */
87 #define ISC_LEXOPT_NOMORE 0x40 /*%< Want "no more" token. */
89 #define ISC_LEXOPT_CNUMBER 0x80 /*%< Recognize octal and hex. */
90 #define ISC_LEXOPT_ESCAPE 0x100 /*%< Recognize escapes. */
91 #define ISC_LEXOPT_QSTRINGMULTILINE 0x200 /*%< Allow multiline "" strings */
92 #define ISC_LEXOPT_OCTAL 0x400 /*%< Expect a octal number. */
96 * Various commenting styles, which may be changed at any time with
97 * isc_lex_setcomments().
100 #define ISC_LEXCOMMENT_C 0x01
101 #define ISC_LEXCOMMENT_CPLUSPLUS 0x02
102 #define ISC_LEXCOMMENT_SHELL 0x04
103 #define ISC_LEXCOMMENT_DNSMASTERFILE 0x08
112 typedef char isc_lexspecials_t[256];
117 isc_tokentype_unknown = 0,
118 isc_tokentype_string = 1,
119 isc_tokentype_number = 2,
120 isc_tokentype_qstring = 3,
121 isc_tokentype_eol = 4,
122 isc_tokentype_eof = 5,
123 isc_tokentype_initialws = 6,
124 isc_tokentype_special = 7,
125 isc_tokentype_nomore = 8
130 unsigned long as_ulong;
131 isc_region_t as_region;
132 isc_textregion_t as_textregion;
136 typedef struct isc_token {
137 isc_tokentype_t type;
138 isc_tokenvalue_t value;
146 isc_lex_create(isc_mem_t *mctx, size_t max_token, isc_lex_t **lexp);
150 * 'max_token' is a hint of the number of bytes in the largest token.
153 *\li '*lexp' is a valid lexer.
158 *\li On success, *lexp is attached to the newly created lexer.
166 isc_lex_destroy(isc_lex_t **lexp);
171 *\li '*lexp' is a valid lexer.
178 isc_lex_getcomments(isc_lex_t *lex);
180 * Return the current lexer commenting styles.
183 *\li 'lex' is a valid lexer.
186 *\li The commenting sytles which are currently allowed.
190 isc_lex_setcomments(isc_lex_t *lex, unsigned int comments);
192 * Set allowed lexer commenting styles.
195 *\li 'lex' is a valid lexer.
197 *\li 'comments' has meaningful values.
201 isc_lex_getspecials(isc_lex_t *lex, isc_lexspecials_t specials);
203 * Put the current list of specials into 'specials'.
206 *\li 'lex' is a valid lexer.
210 isc_lex_setspecials(isc_lex_t *lex, isc_lexspecials_t specials);
212 * The characters in 'specials' are returned as tokens. Along with
213 * whitespace, they delimit strings and numbers.
216 *\li Comment processing takes precedence over special character
220 *\li 'lex' is a valid lexer.
224 isc_lex_openfile(isc_lex_t *lex, const char *filename);
226 * Open 'filename' and make it the current input source for 'lex'.
229 *\li 'lex' is a valid lexer.
231 *\li filename is a valid C string.
235 *\li #ISC_R_NOMEMORY Out of memory
236 *\li #ISC_R_NOTFOUND File not found
237 *\li #ISC_R_NOPERM No permission to open file
238 *\li #ISC_R_FAILURE Couldn't open file, not sure why
239 *\li #ISC_R_UNEXPECTED
243 isc_lex_openstream(isc_lex_t *lex, FILE *stream);
245 * Make 'stream' the current input source for 'lex'.
248 *\li 'lex' is a valid lexer.
250 *\li 'stream' is a valid C stream.
254 *\li #ISC_R_NOMEMORY Out of memory
258 isc_lex_openbuffer(isc_lex_t *lex, isc_buffer_t *buffer);
260 * Make 'buffer' the current input source for 'lex'.
263 *\li 'lex' is a valid lexer.
265 *\li 'buffer' is a valid buffer.
269 *\li #ISC_R_NOMEMORY Out of memory
273 isc_lex_close(isc_lex_t *lex);
275 * Close the most recently opened object (i.e. file or buffer).
279 *\li #ISC_R_NOMORE No more input sources
283 isc_lex_gettoken(isc_lex_t *lex, unsigned int options, isc_token_t *tokenp);
285 * Get the next token.
288 *\li 'lex' is a valid lexer.
290 *\li 'lex' has an input source.
292 *\li 'options' contains valid options.
294 *\li '*tokenp' is a valid pointer.
298 *\li #ISC_R_UNEXPECTEDEND
301 * These two results are returned only if their corresponding lexer
302 * options are not set.
304 *\li #ISC_R_EOF End of input source
305 *\li #ISC_R_NOMORE No more input sources
309 isc_lex_getmastertoken(isc_lex_t *lex, isc_token_t *token,
310 isc_tokentype_t expect, isc_boolean_t eol);
312 * Get the next token from a DNS master file type stream. This is a
313 * convenience function that sets appropriate options and handles quoted
314 * strings and end of line correctly for master files. It also ungets
318 *\li 'lex' is a valid lexer.
320 *\li 'token' is a valid pointer
324 * \li any return code from isc_lex_gettoken().
328 isc_lex_getoctaltoken(isc_lex_t *lex, isc_token_t *token, isc_boolean_t eol);
330 * Get the next token from a DNS master file type stream. This is a
331 * convenience function that sets appropriate options and handles end
332 * of line correctly for master files. It also ungets unexpected tokens.
335 *\li 'lex' is a valid lexer.
337 *\li 'token' is a valid pointer
341 * \li any return code from isc_lex_gettoken().
345 isc_lex_ungettoken(isc_lex_t *lex, isc_token_t *tokenp);
347 * Unget the current token.
350 *\li 'lex' is a valid lexer.
352 *\li 'lex' has an input source.
354 *\li 'tokenp' points to a valid token.
356 *\li There is no ungotten token already.
360 isc_lex_getlasttokentext(isc_lex_t *lex, isc_token_t *tokenp, isc_region_t *r);
362 * Returns a region containing the text of the last token returned.
365 *\li 'lex' is a valid lexer.
367 *\li 'lex' has an input source.
369 *\li 'tokenp' points to a valid token.
371 *\li A token has been gotten and not ungotten.
375 isc_lex_getsourcename(isc_lex_t *lex);
377 * Return the input source name.
380 *\li 'lex' is a valid lexer.
383 * \li source name or NULL if no current source.
384 *\li result valid while current input source exists.
389 isc_lex_getsourceline(isc_lex_t *lex);
391 * Return the input source line number.
394 *\li 'lex' is a valid lexer.
397 *\li Current line number or 0 if no current source.
401 isc_lex_setsourcename(isc_lex_t *lex, const char *name);
403 * Assigns a new name to the input source.
407 * \li 'lex' is a valid lexer.
411 * \li #ISC_R_NOMEMORY
412 * \li #ISC_R_NOTFOUND - there are no sources.
416 isc_lex_isfile(isc_lex_t *lex);
418 * Return whether the current input source is a file.
421 *\li 'lex' is a valid lexer.
424 * \li #ISC_TRUE if the current input is a file,
425 *\li #ISC_FALSE otherwise.
431 #endif /* ISC_LEX_H */