Mercurial > lcfOS
comparison python/baselex.py @ 319:8d07a4254f04
Work on burg
| author | Windel Bouwman |
|---|---|
| date | Sat, 18 Jan 2014 18:58:43 +0100 |
| parents | |
| children | 818be710e13d |
comparison
equal
deleted
inserted
replaced
| 318:e84047f29c78 | 319:8d07a4254f04 |
|---|---|
| 1 | |
| 2 import re | |
| 3 from ppci import Token | |
| 4 | |
| 5 def tokenize(tok_spec, txt): | |
| 6 tok_re = '|'.join('(?P<{}>{})'.format(pair[0], pair[1]) for pair in tok_spec) | |
| 7 gettok = re.compile(tok_re).match | |
| 8 func_map = {pair[0]: pair[2] for pair in tok_spec} | |
| 9 | |
| 10 # Parse line: | |
| 11 line = txt | |
| 12 mo = gettok(line) | |
| 13 pos = 0 | |
| 14 while mo: | |
| 15 typ = mo.lastgroup | |
| 16 val = mo.group(typ) | |
| 17 func = func_map[typ] | |
| 18 if func: | |
| 19 typ, val = func(typ, val) | |
| 20 yield Token(typ, val) | |
| 21 pos = mo.end() | |
| 22 mo = gettok(line, pos) | |
| 23 if len(line) != pos: | |
| 24 raise ParserException('Lex fault at {}'.format(line[pos:])) |