/* ** $Id: llex.c,v 2.20.1.1 2007/12/27 13:02:25 roberto Exp $ ** Lexical Analyzer ** See Copyright Notice in lua.h */ using System; using System.Collections.Generic; using System.Text; using System.Runtime.InteropServices; using System.Diagnostics; namespace KopiLua { using TValue = Lua.lua_TValue; using lua_Number = System.Double; using ZIO = Lua.Zio; public partial class Lua { public const int FIRST_RESERVED = 257; /* maximum length of a reserved word */ public const int TOKEN_LEN = 9; // "function" /* * WARNING: if you change the order of this enumeration, * grep "ORDER RESERVED" */ public enum RESERVED { /* terminal symbols denoted by reserved words */ TK_AND = FIRST_RESERVED, TK_BREAK, TK_DO, TK_ELSE, TK_ELSEIF, TK_END, TK_FALSE, TK_FOR, TK_FUNCTION, TK_IF, TK_IN, TK_LOCAL, TK_NIL, TK_NOT, TK_OR, TK_REPEAT, TK_RETURN, TK_THEN, TK_TRUE, TK_UNTIL, TK_WHILE, /* other terminal symbols */ TK_CONCAT, TK_DOTS, TK_EQ, TK_GE, TK_LE, TK_NE, TK_NUMBER, TK_NAME, TK_STRING, TK_EOS }; /* number of reserved words */ public const int NUM_RESERVED = (int)RESERVED.TK_WHILE - FIRST_RESERVED + 1; public class SemInfo { public SemInfo() { } public SemInfo(SemInfo copy) { this.r = copy.r; this.ts = copy.ts; } public lua_Number r; public TString ts; } ; /* semantics information */ public class Token { public Token() { } public Token(Token copy) { this.token = copy.token; this.seminfo = new SemInfo(copy.seminfo); } public int token; public SemInfo seminfo = new SemInfo(); }; public class LexState { public int current; /* current character (charint) */ public int linenumber; /* input line counter */ public int lastline; /* line of last token `consumed' */ public Token t = new Token(); /* current token */ public Token lookahead = new Token(); /* look ahead token */ public FuncState fs; /* `FuncState' is private to the parser */ public lua_State L; public ZIO z; /* input stream */ public Mbuffer buff; /* buffer for tokens */ public TString source; /* current source name */ public char decpoint; /* locale decimal point */ }; public static void next(LexState ls) { ls.current = zgetc(ls.z); } public static bool currIsNewline(LexState ls) { return (ls.current == '\n' || ls.current == '\r'); } /* ORDER RESERVED */ public static readonly string[] luaX_tokens = { "and", "break", "do", "else", "elseif", "end", "false", "for", "function", "if", "in", "local", "nil", "not", "or", "repeat", "return", "then", "true", "until", "while", "..", "...", "==", ">=", "<=", "~=", "", "", "", "" }; public static void save_and_next(LexState ls) {save(ls, ls.current); next(ls);} private static void save (LexState ls, int c) { Mbuffer b = ls.buff; if (b.n + 1 > b.buffsize) { uint newsize; if (b.buffsize >= MAX_SIZET/2) luaX_lexerror(ls, "lexical element too long", 0); newsize = b.buffsize * 2; luaZ_resizebuffer(ls.L, b, (int)newsize); } b.buffer[b.n++] = (char)c; } public static void luaX_init (lua_State L) { int i; for (i=0; i= MAX_INT) luaX_syntaxerror(ls, "chunk has too many lines"); } public static void luaX_setinput (lua_State L, LexState ls, ZIO z, TString source) { ls.decpoint = '.'; ls.L = L; ls.lookahead.token = (int)RESERVED.TK_EOS; /* no look-ahead token */ ls.z = z; ls.fs = null; ls.linenumber = 1; ls.lastline = 1; ls.source = source; luaZ_resizebuffer(ls.L, ls.buff, LUA_MINBUFFER); /* initialize buffer */ next(ls); /* read first char */ } /* ** ======================================================= ** LEXICAL ANALYZER ** ======================================================= */ private static int check_next (LexState ls, CharPtr set) { if (strchr(set, (char)ls.current) == null) return 0; save_and_next(ls); return 1; } private static void buffreplace (LexState ls, char from, char to) { uint n = luaZ_bufflen(ls.buff); CharPtr p = luaZ_buffer(ls.buff); while ((n--) != 0) if (p[n] == from) p[n] = to; } private static void trydecpoint (LexState ls, SemInfo seminfo) { /* format error: try to update decimal point separator */ // todo: add proper support for localeconv - mjf //lconv cv = localeconv(); char old = ls.decpoint; ls.decpoint = '.'; // (cv ? cv.decimal_point[0] : '.'); buffreplace(ls, old, ls.decpoint); /* try updated decimal separator */ if (luaO_str2d(luaZ_buffer(ls.buff), out seminfo.r) == 0) { /* format error with correct decimal point: no more options */ buffreplace(ls, ls.decpoint, '.'); /* undo change (for error message) */ luaX_lexerror(ls, "malformed number", (int)RESERVED.TK_NUMBER); } } /* LUA_NUMBER */ private static void read_numeral (LexState ls, SemInfo seminfo) { lua_assert(isdigit(ls.current)); do { save_and_next(ls); } while (isdigit(ls.current) || ls.current == '.'); if (check_next(ls, "Ee") != 0) /* `E'? */ check_next(ls, "+-"); /* optional exponent sign */ while (isalnum(ls.current) || ls.current == '_') save_and_next(ls); save(ls, '\0'); buffreplace(ls, '.', ls.decpoint); /* follow locale for decimal point */ if (luaO_str2d(luaZ_buffer(ls.buff), out seminfo.r) == 0) /* format error? */ trydecpoint(ls, seminfo); /* try to update decimal point separator */ } private static int skip_sep (LexState ls) { int count = 0; int s = ls.current; lua_assert(s == '[' || s == ']'); save_and_next(ls); while (ls.current == '=') { save_and_next(ls); count++; } return (ls.current == s) ? count : (-count) - 1; } private static void read_long_string (LexState ls, SemInfo seminfo, int sep) { //int cont = 0; //(void)(cont); /* avoid warnings when `cont' is not used */ save_and_next(ls); /* skip 2nd `[' */ if (currIsNewline(ls)) /* string starts with a newline? */ inclinenumber(ls); /* skip it */ for (;;) { switch (ls.current) { case EOZ: luaX_lexerror(ls, (seminfo != null) ? "unfinished long string" : "unfinished long comment", (int)RESERVED.TK_EOS); break; /* to avoid warnings */ #if LUA_COMPAT_LSTR case '[': { if (skip_sep(ls) == sep) { save_and_next(ls); /* skip 2nd `[' */ cont++; #if LUA_COMPAT_LSTR if (sep == 0) luaX_lexerror(ls, "nesting of [[...]] is deprecated", '['); #endif } break; } #endif case ']': if (skip_sep(ls) == sep) { save_and_next(ls); /* skip 2nd `]' */ //#if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2 // cont--; // if (sep == 0 && cont >= 0) break; //#endif goto endloop; } break; case '\n': case '\r': save(ls, '\n'); inclinenumber(ls); if (seminfo == null) luaZ_resetbuffer(ls.buff); /* avoid wasting space */ break; default: { if (seminfo != null) save_and_next(ls); else next(ls); } break; } } endloop: if (seminfo != null) { seminfo.ts = luaX_newstring(ls, luaZ_buffer(ls.buff) + (2 + sep), (uint)(luaZ_bufflen(ls.buff) - 2*(2 + sep))); } } static void read_string (LexState ls, int del, SemInfo seminfo) { save_and_next(ls); while (ls.current != del) { switch (ls.current) { case EOZ: luaX_lexerror(ls, "unfinished string", (int)RESERVED.TK_EOS); continue; /* to avoid warnings */ case '\n': case '\r': luaX_lexerror(ls, "unfinished string", (int)RESERVED.TK_STRING); continue; /* to avoid warnings */ case '\\': { int c; next(ls); /* do not save the `\' */ switch (ls.current) { case 'a': c = '\a'; break; case 'b': c = '\b'; break; case 'f': c = '\f'; break; case 'n': c = '\n'; break; case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'v': c = '\v'; break; case '\n': /* go through */ case '\r': save(ls, '\n'); inclinenumber(ls); continue; case EOZ: continue; /* will raise an error next loop */ default: { if (!isdigit(ls.current)) save_and_next(ls); /* handles \\, \", \', and \? */ else { /* \xxx */ int i = 0; c = 0; do { c = 10*c + (ls.current-'0'); next(ls); } while (++i<3 && isdigit(ls.current)); if (c > System.Byte.MaxValue) luaX_lexerror(ls, "escape sequence too large", (int)RESERVED.TK_STRING); save(ls, c); } continue; } } save(ls, c); next(ls); continue; } default: save_and_next(ls); break; } } save_and_next(ls); /* skip delimiter */ seminfo.ts = luaX_newstring(ls, luaZ_buffer(ls.buff) + 1, luaZ_bufflen(ls.buff) - 2); } private static int llex (LexState ls, SemInfo seminfo) { luaZ_resetbuffer(ls.buff); for (;;) { switch (ls.current) { case '\n': case '\r': { inclinenumber(ls); continue; } case '-': { next(ls); if (ls.current != '-') return '-'; /* else is a comment */ next(ls); if (ls.current == '[') { int sep = skip_sep(ls); luaZ_resetbuffer(ls.buff); /* `skip_sep' may dirty the buffer */ if (sep >= 0) { read_long_string(ls, null, sep); /* long comment */ luaZ_resetbuffer(ls.buff); continue; } } /* else short comment */ while (!currIsNewline(ls) && ls.current != EOZ) next(ls); continue; } case '[': { int sep = skip_sep(ls); if (sep >= 0) { read_long_string(ls, seminfo, sep); return (int)RESERVED.TK_STRING; } else if (sep == -1) return '['; else luaX_lexerror(ls, "invalid long string delimiter", (int)RESERVED.TK_STRING); } break; case '=': { next(ls); if (ls.current != '=') return '='; else { next(ls); return (int)RESERVED.TK_EQ; } } case '<': { next(ls); if (ls.current != '=') return '<'; else { next(ls); return (int)RESERVED.TK_LE; } } case '>': { next(ls); if (ls.current != '=') return '>'; else { next(ls); return (int)RESERVED.TK_GE; } } case '~': { next(ls); if (ls.current != '=') return '~'; else { next(ls); return (int)RESERVED.TK_NE; } } case '"': case '\'': { read_string(ls, ls.current, seminfo); return (int)RESERVED.TK_STRING; } case '.': { save_and_next(ls); if (check_next(ls, ".") != 0) { if (check_next(ls, ".") != 0) return (int)RESERVED.TK_DOTS; /* ... */ else return (int)RESERVED.TK_CONCAT; /* .. */ } else if (!isdigit(ls.current)) return '.'; else { read_numeral(ls, seminfo); return (int)RESERVED.TK_NUMBER; } } case EOZ: { return (int)RESERVED.TK_EOS; } default: { if (isspace(ls.current)) { lua_assert(!currIsNewline(ls)); next(ls); continue; } else if (isdigit(ls.current)) { read_numeral(ls, seminfo); return (int)RESERVED.TK_NUMBER; } else if (isalpha(ls.current) || ls.current == '_') { /* identifier or reserved word */ TString ts; do { save_and_next(ls); } while (isalnum(ls.current) || ls.current == '_'); ts = luaX_newstring(ls, luaZ_buffer(ls.buff), luaZ_bufflen(ls.buff)); if (ts.tsv.reserved > 0) /* reserved word? */ return ts.tsv.reserved - 1 + FIRST_RESERVED; else { seminfo.ts = ts; return (int)RESERVED.TK_NAME; } } else { int c = ls.current; next(ls); return c; /* single-char tokens (+ - / ...) */ } } } } } public static void luaX_next (LexState ls) { ls.lastline = ls.linenumber; if (ls.lookahead.token != (int)RESERVED.TK_EOS) { /* is there a look-ahead token? */ ls.t = new Token(ls.lookahead); /* use this one */ ls.lookahead.token = (int)RESERVED.TK_EOS; /* and discharge it */ } else ls.t.token = llex(ls, ls.t.seminfo); /* read next token */ } public static void luaX_lookahead (LexState ls) { lua_assert(ls.lookahead.token == (int)RESERVED.TK_EOS); ls.lookahead.token = llex(ls, ls.lookahead.seminfo); } } }