2 * Copyright (c) 2011, 2012
3 * Zhihao Yuan. All rights reserved.
5 * See the LICENSE file for redistribution information.
9 static const char sccsid[] = "$Id: encoding.c,v 1.4 2011/12/13 19:40:52 zy Exp $";
12 #include <sys/types.h>
14 int looks_utf8(const char *, size_t);
15 int looks_utf16(const char *, size_t);
16 int decode_utf8(const char *);
17 int decode_utf16(const char *, int);
19 #define F 0 /* character never appears in text */
20 #define T 1 /* character appears in plain ASCII text */
21 #define I 2 /* character appears in ISO-8859 text */
22 #define X 3 /* character appears in non-ISO extended ASCII (Mac, IBM PC) */
24 static char text_chars[256] = {
25 /* BEL BS HT LF FF CR */
26 F, F, F, F, F, F, F, T, T, T, T, F, T, T, F, F, /* 0x0X */
28 F, F, F, F, F, F, F, F, F, F, F, T, F, F, F, F, /* 0x1X */
29 T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x2X */
30 T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x3X */
31 T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x4X */
32 T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x5X */
33 T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x6X */
34 T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, F, /* 0x7X */
36 X, X, X, X, X, T, X, X, X, X, X, X, X, X, X, X, /* 0x8X */
37 X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, /* 0x9X */
38 I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xaX */
39 I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xbX */
40 I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xcX */
41 I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xdX */
42 I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xeX */
43 I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I /* 0xfX */
48 * Decide whether some text looks like UTF-8. Returns:
51 * 0: uses odd control characters, so doesn't look like text
53 * 2: definitely UTF-8 text (valid high-bit set bytes)
55 * Based on RFC 3629. UTF-8 with BOM is not accepted.
57 * PUBLIC: int looks_utf8(const char *, size_t);
60 looks_utf8(const char *ibuf, size_t nbytes)
62 const u_char *buf = (u_char *)ibuf;
65 int gotone = 0, ctrl = 0;
67 for (i = 0; i < nbytes; i++) {
68 if ((buf[i] & 0x80) == 0) { /* 0xxxxxxx is plain ASCII */
70 * Even if the whole file is valid UTF-8 sequences,
71 * still reject it if it uses weird control characters.
74 if (text_chars[buf[i]] != T)
76 } else if ((buf[i] & 0x40) == 0) { /* 10xxxxxx never 1st byte */
78 } else { /* 11xxxxxx begins UTF-8 */
81 if ((buf[i] & 0x20) == 0) /* 110xxxxx */
82 if (buf[i] > 0xC1) /* C0, C1 */
85 else if ((buf[i] & 0x10) == 0) /* 1110xxxx */
87 else if ((buf[i] & 0x08) == 0) /* 11110xxx */
90 else return -1; /* F5, F6, F7 */
92 return -1; /* F8~FF */
94 for (n = 0; n < following; n++) {
99 if (buf[i] & 0x40) /* 10xxxxxx */
107 return ctrl ? 0 : (gotone ? 2 : 1);
112 * Decide whether some text looks like UTF-16. Returns:
115 * 1: Little-endian UTF-16
116 * 2: Big-endian UTF-16
118 * PUBLIC: int looks_utf16(const char *, size_t);
121 looks_utf16(const char *ibuf, size_t nbytes)
123 const u_char *buf = (u_char *)ibuf;
133 bom = buf[0] << 8 ^ buf[1];
136 else if (bom == 0xFEFF)
141 for (i = 2; i + 1 < nbytes; i += 2) {
143 c = buf[i] << 8 ^ buf[i + 1];
145 c = buf[i] ^ buf[i + 1] << 8;
148 if (c < 0xD800 || c > 0xDFFF)
149 if (c < 128 && text_chars[c] != T)
159 else if (c < 0xDC00 || c > 0xDFFF)
173 * Decode a UTF-8 character from byte string to Unicode.
174 * Returns -1 if the first byte is a not UTF-8 leader.
176 * Based on RFC 3629, but without error detection.
178 * PUBLIC: int decode_utf8(const char *);
181 decode_utf8(const char *ibuf)
183 const u_char *buf = (u_char *)ibuf;
186 if ((buf[0] & 0x80) == 0)
188 else if ((buf[0] & 0x40) == 0);
190 if ((buf[0] & 0x20) == 0)
191 u = (buf[0] ^ 0xC0) << 6 ^ (buf[1] ^ 0x80);
192 else if ((buf[0] & 0x10) == 0)
193 u = (buf[0] ^ 0xE0) << 12 ^ (buf[1] ^ 0x80) << 6
195 else if (((buf[0] & 0x08) == 0))
196 u = (buf[0] ^ 0xF0) << 18 ^ (buf[1] ^ 0x80) << 12
197 ^ (buf[2] ^ 0x80) << 6 ^ (buf[3] ^ 0x80);
205 * Decode a UTF-16 character from byte string to Unicode.
206 * Returns -1 if the first unsigned integer is invalid.
208 * No error detection on supplementary bytes.
210 * PUBLIC: int decode_utf16(const char *, int);
213 decode_utf16(const char* ibuf, int bigend)
215 const u_char *buf = (u_char *)ibuf;
220 w1 = buf[0] << 8 ^ buf[1];
222 w1 = buf[0] ^ buf[1] << 8;
224 if (w1 < 0xD800 || w1 > 0xDFFF)
226 else if (w1 > 0xDBFF);
229 w2 = buf[2] << 8 ^ buf[3];
231 w2 = buf[2] ^ buf[3] << 8;
232 u = ((w1 ^ 0xD800) << 10 ^ (w2 ^ 0xDC00)) + 0x10000;