3 * Tokenize a string, accommodating quoted strings.
9 * This file defines the string_tokenize interface
10 * This file is part of AutoOpts, a companion to AutoGen.
11 * AutoOpts is free software.
12 * AutoOpts is Copyright (C) 1992-2015 by Bruce Korb - all rights reserved
14 * AutoOpts is available under any one of two licenses. The license
15 * in use must be one of these two and the choice is under the control
16 * of the user of the license.
18 * The GNU Lesser General Public License, version 3 or later
19 * See the files "COPYING.lgplv3" and "COPYING.gplv3"
21 * The Modified Berkeley Software Distribution License
22 * See the file "COPYING.mbsd"
24 * These files have the following sha256 sums:
26 * 8584710e9b04216a394078dc156b781d0b47e1729104d666658aecef8ee32e95 COPYING.gplv3
27 * 4379e7444a0e2ce2b12dd6f5a52a27a4d02d39d247901d3285c88cf0d37f477b COPYING.lgplv3
28 * 13aa749a5b0a454917a944ed8fffc530b784f5ead522b1aacaf4ec8aa55a6239 COPYING.mbsd
34 #define cc_t const unsigned char
35 #define ch_t unsigned char
37 /* = = = START-STATIC-FORWARD = = = */
39 copy_cooked(ch_t ** ppDest, char const ** ppSrc);
42 copy_raw(ch_t ** ppDest, char const ** ppSrc);
45 alloc_token_list(char const * str);
46 /* = = = END-STATIC-FORWARD = = = */
49 copy_cooked(ch_t ** ppDest, char const ** ppSrc)
51 ch_t * pDest = (ch_t *)*ppDest;
52 const ch_t * pSrc = (const ch_t *)(*ppSrc + 1);
57 case NUL: *ppSrc = NULL; return;
60 pSrc += ao_string_cook_escape_char((char *)pSrc, (char *)&ch, 0x7F);
71 *ppDest = (ch_t *)pDest; /* next spot for storing character */
72 *ppSrc = (char const *)pSrc; /* char following closing quote */
77 copy_raw(ch_t ** ppDest, char const ** ppSrc)
79 ch_t * pDest = *ppDest;
80 cc_t * pSrc = (cc_t *) (*ppSrc + 1);
85 case NUL: *ppSrc = NULL; return;
89 * *Four* escapes are handled: newline removal, escape char
90 * quoting and apostrophe quoting
93 case NUL: *ppSrc = NULL; return;
119 *ppDest = pDest; /* next spot for storing character */
120 *ppSrc = (char const *) pSrc; /* char following closing quote */
123 static token_list_t *
124 alloc_token_list(char const * str)
128 int max_token_ct = 2; /* allow for trailing NULL pointer & NUL on string */
130 if (str == NULL) goto enoent_res;
133 * Trim leading white space. Use "ENOENT" and a NULL return to indicate
134 * an empty string was passed.
136 str = SPN_WHITESPACE_CHARS(str);
137 if (*str == NUL) goto enoent_res;
140 * Take an approximate count of tokens. If no quoted strings are used,
141 * it will be accurate. If quoted strings are used, it will be a little
142 * high and we'll squander the space for a few extra pointers.
145 char const * pz = str;
149 pz = BRK_WHITESPACE_CHARS(pz+1);
150 pz = SPN_WHITESPACE_CHARS(pz);
151 } while (*pz != NUL);
153 res = malloc(sizeof(*res) + (size_t)(pz - str)
154 + ((size_t)max_token_ct * sizeof(ch_t *)));
159 else res->tkn_list[0] = (ch_t *)(res->tkn_list + (max_token_ct - 1));
169 /*=export_func ao_string_tokenize
171 * what: tokenize an input string
173 * arg: + char const * + string + string to be tokenized +
175 * ret_type: token_list_t *
176 * ret_desc: pointer to a structure that lists each token
180 * This function will convert one input string into a list of strings.
181 * The list of strings is derived by separating the input based on
182 * white space separation. However, if the input contains either single
183 * or double quote characters, then the text after that character up to
184 * a matching quote will become the string in the list.
186 * The returned pointer should be deallocated with @code{free(3C)} when
187 * are done using the data. The data are placed in a single block of
188 * allocated memory. Do not deallocate individual token/strings.
190 * The structure pointed to will contain at least these two fields:
193 * The number of tokens found in the input string.
195 * An array of @code{tkn_ct + 1} pointers to substring tokens, with
196 * the last pointer set to NULL.
199 * There are two types of quoted strings: single quoted (@code{'}) and
200 * double quoted (@code{"}). Singly quoted strings are fairly raw in that
201 * escape characters (@code{\\}) are simply another character, except when
202 * preceding the following characters:
204 * @code{\\} double backslashes reduce to one
205 * @code{'} incorporates the single quote into the string
206 * @code{\n} suppresses both the backslash and newline character
209 * Double quote strings are formed according to the rules of string
210 * constants in ANSI-C programs.
214 * #include <stdlib.h>
216 * token_list_t * ptl = ao_string_tokenize(some_string)
217 * for (ix = 0; ix < ptl->tkn_ct; ix++)
218 * do_something_with_tkn(ptl->tkn_list[ix]);
221 * Note that everything is freed with the one call to @code{free(3C)}.
224 * NULL is returned and @code{errno} will be set to indicate the problem:
227 * @code{EINVAL} - There was an unterminated quoted string.
229 * @code{ENOENT} - The input string was empty.
231 * @code{ENOMEM} - There is not enough memory.
235 ao_string_tokenize(char const * str)
237 token_list_t * res = alloc_token_list(str);
241 * Now copy each token into the output buffer.
246 pzDest = (ch_t *)(res->tkn_list[0]);
250 res->tkn_list[ res->tkn_ct++ ] = pzDest;
253 if (IS_WHITESPACE_CHAR(ch)) {
255 str = SPN_WHITESPACE_CHARS(str+1);
261 copy_cooked(&pzDest, &str);
267 if (IS_WHITESPACE_CHAR(*str))
268 goto found_white_space;
272 copy_raw(&pzDest, &str);
278 if (IS_WHITESPACE_CHAR(*str))
279 goto found_white_space;
287 *(pzDest++) = (unsigned char)ch;
292 * NUL terminate the last token and see if we have any more tokens.
295 } while (*str != NUL);
297 res->tkn_list[ res->tkn_ct ] = NULL;
307 main(int argc, char ** argv)
310 printf("USAGE: %s arg [ ... ]\n", *argv);
314 char * arg = *(++argv);
315 token_list_t * p = ao_string_tokenize(arg);
317 printf("Parsing string ``%s'' failed:\n\terrno %d (%s)\n",
318 arg, errno, strerror(errno));
321 printf("Parsed string ``%s''\ninto %d tokens:\n", arg, p->tkn_ct);
323 printf(" %3d: ``%s''\n", ix+1, p->tkn_list[ix]);
324 } while (++ix < p->tkn_ct);
336 * c-file-style: "stroustrup"
337 * indent-tabs-mode: nil
339 * end of autoopts/tokenize.c */