Mercurial > mm7

comparison lib/lua/lua-5.2.2/llex.c @ 1866:41cc4dd3c122

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Lua 5.2.2 added.
author Nomad
date Wed, 16 Oct 2013 13:34:26 +0200
parents
children
comparison
equal deleted inserted replaced
1848:3b39b70e8e93 1866:41cc4dd3c122
1 /*
2 ** $Id: llex.c,v 2.63 2013/03/16 21:10:18 roberto Exp $
3 ** Lexical Analyzer
4 ** See Copyright Notice in lua.h
5 */
6
7
8 #include <locale.h>
9 #include <string.h>
10
11 #define llex_c
12 #define LUA_CORE
13
14 #include "lua.h"
15
16 #include "lctype.h"
17 #include "ldo.h"
18 #include "llex.h"
19 #include "lobject.h"
20 #include "lparser.h"
21 #include "lstate.h"
22 #include "lstring.h"
23 #include "ltable.h"
24 #include "lzio.h"
25
26
27
28 #define next(ls) (ls->current = zgetc(ls->z))
29
30
31
32 #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r')
33
34
35 /* ORDER RESERVED */
36 static const char *const luaX_tokens [] = {
37 "and", "break", "do", "else", "elseif",
38 "end", "false", "for", "function", "goto", "if",
39 "in", "local", "nil", "not", "or", "repeat",
40 "return", "then", "true", "until", "while",
41 "..", "...", "==", ">=", "<=", "~=", "::", "<eof>",
42 "<number>", "<name>", "<string>"
43 };
44
45
46 #define save_and_next(ls) (save(ls, ls->current), next(ls))
47
48
49 static l_noret lexerror (LexState *ls, const char *msg, int token);
50
51
52 static void save (LexState *ls, int c) {
53 Mbuffer *b = ls->buff;
54 if (luaZ_bufflen(b) + 1 > luaZ_sizebuffer(b)) {
55 size_t newsize;
56 if (luaZ_sizebuffer(b) >= MAX_SIZET/2)
57 lexerror(ls, "lexical element too long", 0);
58 newsize = luaZ_sizebuffer(b) * 2;
59 luaZ_resizebuffer(ls->L, b, newsize);
60 }
61 b->buffer[luaZ_bufflen(b)++] = cast(char, c);
62 }
63
64
65 void luaX_init (lua_State *L) {
66 int i;
67 for (i=0; i<NUM_RESERVED; i++) {
68 TString *ts = luaS_new(L, luaX_tokens[i]);
69 luaS_fix(ts); /* reserved words are never collected */
70 ts->tsv.extra = cast_byte(i+1); /* reserved word */
71 }
72 }
73
74
75 const char *luaX_token2str (LexState *ls, int token) {
76 if (token < FIRST_RESERVED) { /* single-byte symbols? */
77 lua_assert(token == cast(unsigned char, token));
78 return (lisprint(token)) ? luaO_pushfstring(ls->L, LUA_QL("%c"), token) :
79 luaO_pushfstring(ls->L, "char(%d)", token);
80 }
81 else {
82 const char *s = luaX_tokens[token - FIRST_RESERVED];
83 if (token < TK_EOS) /* fixed format (symbols and reserved words)? */
84 return luaO_pushfstring(ls->L, LUA_QS, s);
85 else /* names, strings, and numerals */
86 return s;
87 }
88 }
89
90
91 static const char *txtToken (LexState *ls, int token) {
92 switch (token) {
93 case TK_NAME:
94 case TK_STRING:
95 case TK_NUMBER:
96 save(ls, '\0');
97 return luaO_pushfstring(ls->L, LUA_QS, luaZ_buffer(ls->buff));
98 default:
99 return luaX_token2str(ls, token);
100 }
101 }
102
103
104 static l_noret lexerror (LexState *ls, const char *msg, int token) {
105 char buff[LUA_IDSIZE];
106 luaO_chunkid(buff, getstr(ls->source), LUA_IDSIZE);
107 msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg);
108 if (token)
109 luaO_pushfstring(ls->L, "%s near %s", msg, txtToken(ls, token));
110 luaD_throw(ls->L, LUA_ERRSYNTAX);
111 }
112
113
114 l_noret luaX_syntaxerror (LexState *ls, const char *msg) {
115 lexerror(ls, msg, ls->t.token);
116 }
117
118
119 /*
120 ** creates a new string and anchors it in function's table so that
121 ** it will not be collected until the end of the function's compilation
122 ** (by that time it should be anchored in function's prototype)
123 */
124 TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
125 lua_State *L = ls->L;
126 TValue *o; /* entry for `str' */
127 TString *ts = luaS_newlstr(L, str, l); /* create new string */
128 setsvalue2s(L, L->top++, ts); /* temporarily anchor it in stack */
129 o = luaH_set(L, ls->fs->h, L->top - 1);
130 if (ttisnil(o)) { /* not in use yet? (see 'addK') */
131 /* boolean value does not need GC barrier;
132 table has no metatable, so it does not need to invalidate cache */
133 setbvalue(o, 1); /* t[string] = true */
134 luaC_checkGC(L);
135 }
136 L->top--; /* remove string from stack */
137 return ts;
138 }
139
140
141 /*
142 ** increment line number and skips newline sequence (any of
143 ** \n, \r, \n\r, or \r\n)
144 */
145 static void inclinenumber (LexState *ls) {
146 int old = ls->current;
147 lua_assert(currIsNewline(ls));
148 next(ls); /* skip `\n' or `\r' */
149 if (currIsNewline(ls) && ls->current != old)
150 next(ls); /* skip `\n\r' or `\r\n' */
151 if (++ls->linenumber >= MAX_INT)
152 luaX_syntaxerror(ls, "chunk has too many lines");
153 }
154
155
156 void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source,
157 int firstchar) {
158 ls->decpoint = '.';
159 ls->L = L;
160 ls->current = firstchar;
161 ls->lookahead.token = TK_EOS; /* no look-ahead token */
162 ls->z = z;
163 ls->fs = NULL;
164 ls->linenumber = 1;
165 ls->lastline = 1;
166 ls->source = source;
167 ls->envn = luaS_new(L, LUA_ENV); /* create env name */
168 luaS_fix(ls->envn); /* never collect this name */
169 luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */
170 }
171
172
173
174 /*
175 ** =======================================================
176 ** LEXICAL ANALYZER
177 ** =======================================================
178 */
179
180
181
182 static int check_next (LexState *ls, const char *set) {
183 if (ls->current == '\0' || !strchr(set, ls->current))
184 return 0;
185 save_and_next(ls);
186 return 1;
187 }
188
189
190 /*
191 ** change all characters 'from' in buffer to 'to'
192 */
193 static void buffreplace (LexState *ls, char from, char to) {
194 size_t n = luaZ_bufflen(ls->buff);
195 char *p = luaZ_buffer(ls->buff);
196 while (n--)
197 if (p[n] == from) p[n] = to;
198 }
199
200
201 #if !defined(getlocaledecpoint)
202 #define getlocaledecpoint() (localeconv()->decimal_point[0])
203 #endif
204
205
206 #define buff2d(b,e) luaO_str2d(luaZ_buffer(b), luaZ_bufflen(b) - 1, e)
207
208 /*
209 ** in case of format error, try to change decimal point separator to
210 ** the one defined in the current locale and check again
211 */
212 static void trydecpoint (LexState *ls, SemInfo *seminfo) {
213 char old = ls->decpoint;
214 ls->decpoint = getlocaledecpoint();
215 buffreplace(ls, old, ls->decpoint); /* try new decimal separator */
216 if (!buff2d(ls->buff, &seminfo->r)) {
217 /* format error with correct decimal point: no more options */
218 buffreplace(ls, ls->decpoint, '.'); /* undo change (for error message) */
219 lexerror(ls, "malformed number", TK_NUMBER);
220 }
221 }
222
223
224 /* LUA_NUMBER */
225 /*
226 ** this function is quite liberal in what it accepts, as 'luaO_str2d'
227 ** will reject ill-formed numerals.
228 */
229 static void read_numeral (LexState *ls, SemInfo *seminfo) {
230 const char *expo = "Ee";
231 int first = ls->current;
232 lua_assert(lisdigit(ls->current));
233 save_and_next(ls);
234 if (first == '0' && check_next(ls, "Xx")) /* hexadecimal? */
235 expo = "Pp";
236 for (;;) {
237 if (check_next(ls, expo)) /* exponent part? */
238 check_next(ls, "+-"); /* optional exponent sign */
239 if (lisxdigit(ls->current) || ls->current == '.')
240 save_and_next(ls);
241 else break;
242 }
243 save(ls, '\0');
244 buffreplace(ls, '.', ls->decpoint); /* follow locale for decimal point */
245 if (!buff2d(ls->buff, &seminfo->r)) /* format error? */
246 trydecpoint(ls, seminfo); /* try to update decimal point separator */
247 }
248
249
250 /*
251 ** skip a sequence '[=*[' or ']=*]' and return its number of '='s or
252 ** -1 if sequence is malformed
253 */
254 static int skip_sep (LexState *ls) {
255 int count = 0;
256 int s = ls->current;
257 lua_assert(s == '[' || s == ']');
258 save_and_next(ls);
259 while (ls->current == '=') {
260 save_and_next(ls);
261 count++;
262 }
263 return (ls->current == s) ? count : (-count) - 1;
264 }
265
266
267 static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
268 save_and_next(ls); /* skip 2nd `[' */
269 if (currIsNewline(ls)) /* string starts with a newline? */
270 inclinenumber(ls); /* skip it */
271 for (;;) {
272 switch (ls->current) {
273 case EOZ:
274 lexerror(ls, (seminfo) ? "unfinished long string" :
275 "unfinished long comment", TK_EOS);
276 break; /* to avoid warnings */
277 case ']': {
278 if (skip_sep(ls) == sep) {
279 save_and_next(ls); /* skip 2nd `]' */
280 goto endloop;
281 }
282 break;
283 }
284 case '\n': case '\r': {
285 save(ls, '\n');
286 inclinenumber(ls);
287 if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */
288 break;
289 }
290 default: {
291 if (seminfo) save_and_next(ls);
292 else next(ls);
293 }
294 }
295 } endloop:
296 if (seminfo)
297 seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
298 luaZ_bufflen(ls->buff) - 2*(2 + sep));
299 }
300
301
302 static void escerror (LexState *ls, int *c, int n, const char *msg) {
303 int i;
304 luaZ_resetbuffer(ls->buff); /* prepare error message */
305 save(ls, '\\');
306 for (i = 0; i < n && c[i] != EOZ; i++)
307 save(ls, c[i]);
308 lexerror(ls, msg, TK_STRING);
309 }
310
311
312 static int readhexaesc (LexState *ls) {
313 int c[3], i; /* keep input for error message */
314 int r = 0; /* result accumulator */
315 c[0] = 'x'; /* for error message */
316 for (i = 1; i < 3; i++) { /* read two hexadecimal digits */
317 c[i] = next(ls);
318 if (!lisxdigit(c[i]))
319 escerror(ls, c, i + 1, "hexadecimal digit expected");
320 r = (r << 4) + luaO_hexavalue(c[i]);
321 }
322 return r;
323 }
324
325
326 static int readdecesc (LexState *ls) {
327 int c[3], i;
328 int r = 0; /* result accumulator */
329 for (i = 0; i < 3 && lisdigit(ls->current); i++) { /* read up to 3 digits */
330 c[i] = ls->current;
331 r = 10*r + c[i] - '0';
332 next(ls);
333 }
334 if (r > UCHAR_MAX)
335 escerror(ls, c, i, "decimal escape too large");
336 return r;
337 }
338
339
340 static void read_string (LexState *ls, int del, SemInfo *seminfo) {
341 save_and_next(ls); /* keep delimiter (for error messages) */
342 while (ls->current != del) {
343 switch (ls->current) {
344 case EOZ:
345 lexerror(ls, "unfinished string", TK_EOS);
346 break; /* to avoid warnings */
347 case '\n':
348 case '\r':
349 lexerror(ls, "unfinished string", TK_STRING);
350 break; /* to avoid warnings */
351 case '\\': { /* escape sequences */
352 int c; /* final character to be saved */
353 next(ls); /* do not save the `\' */
354 switch (ls->current) {
355 case 'a': c = '\a'; goto read_save;
356 case 'b': c = '\b'; goto read_save;
357 case 'f': c = '\f'; goto read_save;
358 case 'n': c = '\n'; goto read_save;
359 case 'r': c = '\r'; goto read_save;
360 case 't': c = '\t'; goto read_save;
361 case 'v': c = '\v'; goto read_save;
362 case 'x': c = readhexaesc(ls); goto read_save;
363 case '\n': case '\r':
364 inclinenumber(ls); c = '\n'; goto only_save;
365 case '\\': case '\"': case '\'':
366 c = ls->current; goto read_save;
367 case EOZ: goto no_save; /* will raise an error next loop */
368 case 'z': { /* zap following span of spaces */
369 next(ls); /* skip the 'z' */
370 while (lisspace(ls->current)) {
371 if (currIsNewline(ls)) inclinenumber(ls);
372 else next(ls);
373 }
374 goto no_save;
375 }
376 default: {
377 if (!lisdigit(ls->current))
378 escerror(ls, &ls->current, 1, "invalid escape sequence");
379 /* digital escape \ddd */
380 c = readdecesc(ls);
381 goto only_save;
382 }
383 }
384 read_save: next(ls); /* read next character */
385 only_save: save(ls, c); /* save 'c' */
386 no_save: break;
387 }
388 default:
389 save_and_next(ls);
390 }
391 }
392 save_and_next(ls); /* skip delimiter */
393 seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
394 luaZ_bufflen(ls->buff) - 2);
395 }
396
397
398 static int llex (LexState *ls, SemInfo *seminfo) {
399 luaZ_resetbuffer(ls->buff);
400 for (;;) {
401 switch (ls->current) {
402 case '\n': case '\r': { /* line breaks */
403 inclinenumber(ls);
404 break;
405 }
406 case ' ': case '\f': case '\t': case '\v': { /* spaces */
407 next(ls);
408 break;
409 }
410 case '-': { /* '-' or '--' (comment) */
411 next(ls);
412 if (ls->current != '-') return '-';
413 /* else is a comment */
414 next(ls);
415 if (ls->current == '[') { /* long comment? */
416 int sep = skip_sep(ls);
417 luaZ_resetbuffer(ls->buff); /* `skip_sep' may dirty the buffer */
418 if (sep >= 0) {
419 read_long_string(ls, NULL, sep); /* skip long comment */
420 luaZ_resetbuffer(ls->buff); /* previous call may dirty the buff. */
421 break;
422 }
423 }
424 /* else short comment */
425 while (!currIsNewline(ls) && ls->current != EOZ)
426 next(ls); /* skip until end of line (or end of file) */
427 break;
428 }
429 case '[': { /* long string or simply '[' */
430 int sep = skip_sep(ls);
431 if (sep >= 0) {
432 read_long_string(ls, seminfo, sep);
433 return TK_STRING;
434 }
435 else if (sep == -1) return '[';
436 else lexerror(ls, "invalid long string delimiter", TK_STRING);
437 }
438 case '=': {
439 next(ls);
440 if (ls->current != '=') return '=';
441 else { next(ls); return TK_EQ; }
442 }
443 case '<': {
444 next(ls);
445 if (ls->current != '=') return '<';
446 else { next(ls); return TK_LE; }
447 }
448 case '>': {
449 next(ls);
450 if (ls->current != '=') return '>';
451 else { next(ls); return TK_GE; }
452 }
453 case '~': {
454 next(ls);
455 if (ls->current != '=') return '~';
456 else { next(ls); return TK_NE; }
457 }
458 case ':': {
459 next(ls);
460 if (ls->current != ':') return ':';
461 else { next(ls); return TK_DBCOLON; }
462 }
463 case '"': case '\'': { /* short literal strings */
464 read_string(ls, ls->current, seminfo);
465 return TK_STRING;
466 }
467 case '.': { /* '.', '..', '...', or number */
468 save_and_next(ls);
469 if (check_next(ls, ".")) {
470 if (check_next(ls, "."))
471 return TK_DOTS; /* '...' */
472 else return TK_CONCAT; /* '..' */
473 }
474 else if (!lisdigit(ls->current)) return '.';
475 /* else go through */
476 }
477 case '0': case '1': case '2': case '3': case '4':
478 case '5': case '6': case '7': case '8': case '9': {
479 read_numeral(ls, seminfo);
480 return TK_NUMBER;
481 }
482 case EOZ: {
483 return TK_EOS;
484 }
485 default: {
486 if (lislalpha(ls->current)) { /* identifier or reserved word? */
487 TString *ts;
488 do {
489 save_and_next(ls);
490 } while (lislalnum(ls->current));
491 ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
492 luaZ_bufflen(ls->buff));
493 seminfo->ts = ts;
494 if (isreserved(ts)) /* reserved word? */
495 return ts->tsv.extra - 1 + FIRST_RESERVED;
496 else {
497 return TK_NAME;
498 }
499 }
500 else { /* single-char tokens (+ - / ...) */
501 int c = ls->current;
502 next(ls);
503 return c;
504 }
505 }
506 }
507 }
508 }
509
510
511 void luaX_next (LexState *ls) {
512 ls->lastline = ls->linenumber;
513 if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */
514 ls->t = ls->lookahead; /* use this one */
515 ls->lookahead.token = TK_EOS; /* and discharge it */
516 }
517 else
518 ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */
519 }
520
521
522 int luaX_lookahead (LexState *ls) {
523 lua_assert(ls->lookahead.token == TK_EOS);
524 ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
525 return ls->lookahead.token;
526 }
527