2 * This file defines the string_tokenize interface
3 * Time-stamp: "2006-06-24 15:27:49 bkorb"
5 * string_tokenize copyright 2005 Bruce Korb
7 * string_tokenize is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
12 * string_tokenize is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with string_tokenize; if not, write to:
19 * The Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor,
21 * Boston, MA 02110-1301, USA.
27 #define cc_t const unsigned char
28 #define ch_t unsigned char
30 /* = = = START-STATIC-FORWARD = = = */
31 /* static forward declarations maintained by :mkfwd */
33 copy_cooked( ch_t** ppDest, char const ** ppSrc );
36 copy_raw( ch_t** ppDest, char const ** ppSrc );
37 /* = = = END-STATIC-FORWARD = = = */
40 copy_cooked( ch_t** ppDest, char const ** ppSrc )
42 ch_t* pDest = (ch_t*)*ppDest;
43 const ch_t* pSrc = (const ch_t*)(*ppSrc + 1);
48 case NUL: *ppSrc = NULL; return;
51 pSrc += ao_string_cook_escape_char( (char*)pSrc, (char*)&ch, 0x7F );
62 *ppDest = (ch_t*)pDest; /* next spot for storing character */
63 *ppSrc = (char const *)pSrc; /* char following closing quote */
68 copy_raw( ch_t** ppDest, char const ** ppSrc )
70 ch_t* pDest = *ppDest;
71 cc_t* pSrc = (cc_t*) (*ppSrc + 1);
76 case NUL: *ppSrc = NULL; return;
80 * *Four* escapes are handled: newline removal, escape char
81 * quoting and apostrophe quoting
84 case NUL: *ppSrc = NULL; return;
86 if (*(++pSrc) == '\n')
110 *ppDest = pDest; /* next spot for storing character */
111 *ppSrc = (char const *) pSrc; /* char following closing quote */
115 /*=export_func ao_string_tokenize
117 * what: tokenize an input string
119 * arg: + char const* + string + string to be tokenized +
121 * ret_type: token_list_t*
122 * ret_desc: pointer to a structure that lists each token
126 * This function will convert one input string into a list of strings.
127 * The list of strings is derived by separating the input based on
128 * white space separation. However, if the input contains either single
129 * or double quote characters, then the text after that character up to
130 * a matching quote will become the string in the list.
132 * The returned pointer should be deallocated with @code{free(3C)} when
133 * are done using the data. The data are placed in a single block of
134 * allocated memory. Do not deallocate individual token/strings.
136 * The structure pointed to will contain at least these two fields:
139 * The number of tokens found in the input string.
141 * An array of @code{tkn_ct + 1} pointers to substring tokens, with
142 * the last pointer set to NULL.
145 * There are two types of quoted strings: single quoted (@code{'}) and
146 * double quoted (@code{"}). Singly quoted strings are fairly raw in that
147 * escape characters (@code{\\}) are simply another character, except when
148 * preceding the following characters:
150 * @code{\\} double backslashes reduce to one
151 * @code{'} incorporates the single quote into the string
152 * @code{\n} suppresses both the backslash and newline character
155 * Double quote strings are formed according to the rules of string
156 * constants in ANSI-C programs.
160 * #include <stdlib.h>
162 * token_list_t* ptl = ao_string_tokenize( some_string )
163 * for (ix = 0; ix < ptl->tkn_ct; ix++)
164 * do_something_with_tkn( ptl->tkn_list[ix] );
167 * Note that everything is freed with the one call to @code{free(3C)}.
170 * NULL is returned and @code{errno} will be set to indicate the problem:
173 * @code{EINVAL} - There was an unterminated quoted string.
175 * @code{ENOENT} - The input string was empty.
177 * @code{ENOMEM} - There is not enough memory.
181 ao_string_tokenize( char const* str )
183 int max_token_ct = 1; /* allow for trailing NUL on string */
186 if (str == NULL) goto bogus_str;
189 * Trim leading white space. Use "ENOENT" and a NULL return to indicate
190 * an empty string was passed.
192 while (isspace( (ch_t)*str )) str++;
200 * Take an approximate count of tokens. If no quoted strings are used,
201 * it will be accurate. If quoted strings are used, it will be a little
202 * high and we'll squander the space for a few extra pointers.
205 cc_t* pz = (cc_t*)str;
209 while (! isspace( *++pz ))
210 if (*pz == NUL) goto found_nul;
211 while (isspace( *pz )) pz++;
212 } while (*pz != NUL);
218 res = malloc( sizeof(*res) + strlen(str) + (max_token_ct * sizeof(ch_t*)) );
225 * Now copy each token into the output buffer.
228 ch_t* pzDest = (ch_t*)(res->tkn_list + (max_token_ct + 1));
232 res->tkn_list[ res->tkn_ct++ ] = pzDest;
237 while (isspace( (ch_t)*++str )) ;
243 copy_cooked( &pzDest, &str );
249 if (isspace( (ch_t)*str ))
250 goto found_white_space;
254 copy_raw( &pzDest, &str );
260 if (isspace( (ch_t)*str ))
261 goto found_white_space;
274 * NUL terminate the last token and see if we have any more tokens.
277 } while (*str != NUL);
279 res->tkn_list[ res->tkn_ct ] = NULL;
290 main( int argc, char** argv )
293 printf("USAGE: %s arg [ ... ]\n", *argv);
297 char* arg = *(++argv);
298 token_list_t* p = ao_string_tokenize( arg );
300 printf( "Parsing string ``%s'' failed:\n\terrno %d (%s)\n",
301 arg, errno, strerror( errno ));
304 printf( "Parsed string ``%s''\ninto %d tokens:\n", arg, p->tkn_ct );
306 printf( " %3d: ``%s''\n", ix+1, p->tkn_list[ix] );
307 } while (++ix < p->tkn_ct);
318 * c-file-style: "stroustrup"
319 * indent-tabs-mode: nil
321 * end of autoopts/tokenize.c */