lcfOS: python/baselex.py comparison

Made lexer more generic

comparison

equal deleted inserted replaced

-:6df89163e114
+:0c44e494ef58
 import re
 from ppci import Token, CompilerError
+from pyyacc import EOF
-def tokenize(tok_spec, txt):
-tok_re = '|'.join('(?P<{}>{})'.format(pair[0], pair[1]) for pair in tok_spec)
-gettok = re.compile(tok_re).match
-func_map = {pair[0]: pair[2] for pair in tok_spec}
-# Parse line:
+class BaseLexer:
-line = txt
+""" Base class for a lexer """
-mo = gettok(line)
+def __init__(self, tok_spec):
-pos = 0
+tok_re = '|'.join('(?P<{}>{})'.format(pair[0], pair[1]) for pair in tok_spec)
-while mo:
+self.gettok = re.compile(tok_re).match
-typ = mo.lastgroup
+self.func_map = {pair[0]: pair[2] for pair in tok_spec}
-val = mo.group(typ)
-func = func_map[typ]
+def feed(self, txt):
-if func:
+""" Feeds the lexer with extra input """
-typ, val = func(typ, val)
+self.tokens = self.tokenize(txt)
-yield Token(typ, val)
-pos = mo.end()
+def tokenize(self, txt):
-mo = gettok(line, pos)
+""" Generator that generates tokens from text """
-if len(line) != pos:
+mo = self.gettok(txt)
-raise CompilerError('Lex fault at {}'.format(line[pos:]))
+pos = 0
+while mo:
+typ = mo.lastgroup
+val = mo.group(typ)
+func = self.func_map[typ]
+if func:
+typ, val = func(typ, val)
+yield Token(typ, val)
+pos = mo.end()
+mo = self.gettok(txt, pos)
+if len(txt) != pos:
+raise CompilerError('Lex fault at {}'.format(txt[pos:]))
+def next_token(self):
+try:
+return self.tokens.__next__()
+except StopIteration:
+return Token(EOF, EOF)

Mercurial > lcfOS