annotate python/ppci/frontends/ks/lexer.py @ 101:af0d7913677a

Fixes and splitting into 3 stage
author windel
date Mon, 24 Dec 2012 17:55:08 +0100
parents 3f772feb12ef
children
rev   line source
101
af0d7913677a Fixes and splitting into 3 stage
windel
parents: 98
diff changeset
1 import collections, re
af0d7913677a Fixes and splitting into 3 stage
windel
parents: 98
diff changeset
2 from ...core.errors import CompilerException
1
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
3
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
4 """
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
5 Lexical analyzer part. Splits the input character stream into tokens.
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
6 """
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
7
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
8 # Token is used in the lexical analyzer:
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
9 Token = collections.namedtuple('Token', 'typ val row col')
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
10
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
11 keywords = ['and', 'array', 'begin', 'by', 'case', 'const', 'div', 'do', \
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
12 'else', 'elsif', 'end', 'false', 'for', 'if', 'import', 'in', 'is', \
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
13 'mod', 'module', 'nil', 'not', 'of', 'or', 'pointer', 'procedure', \
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
14 'record', 'repeat', 'return', 'then', 'to', 'true', 'type', 'until', 'var', \
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
15 'while', 'asm' ]
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
16
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
17 def tokenize(s):
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
18 """
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
19 Tokenizer, generates an iterator that
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
20 returns tokens!
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
21
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
22 This GREAT example was taken from python re doc page!
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
23 """
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
24 tok_spec = [
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
25 ('REAL', r'\d+\.\d+'),
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
26 ('HEXNUMBER', r'0x[\da-fA-F]+'),
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
27 ('NUMBER', r'\d+'),
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
28 ('ID', r'[A-Za-z][A-Za-z\d_]*'),
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
29 ('NEWLINE', r'\n'),
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
30 ('SKIP', r'[ \t]'),
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
31 ('COMMENTS', r'{.*}'),
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
32 ('LEESTEKEN', r':=|[\.,=:;\-+*\[\]/\(\)]|>=|<=|<>|>|<'),
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
33 ('STRING', r"'.*?'")
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
34 ]
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
35 tok_re = '|'.join('(?P<%s>%s)' % pair for pair in tok_spec)
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
36 gettok = re.compile(tok_re).match
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
37 line = 1
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
38 pos = line_start = 0
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
39 mo = gettok(s)
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
40 while mo is not None:
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
41 typ = mo.lastgroup
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
42 val = mo.group(typ)
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
43 if typ == 'NEWLINE':
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
44 line_start = pos
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
45 line += 1
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
46 elif typ == 'COMMENTS':
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
47 pass
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
48 elif typ != 'SKIP':
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
49 if typ == 'ID':
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
50 if val in keywords:
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
51 typ = val
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
52 elif typ == 'LEESTEKEN':
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
53 typ = val
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
54 elif typ == 'NUMBER':
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
55 val = int(val)
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
56 elif typ == 'HEXNUMBER':
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
57 val = int(val[2:], 16)
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
58 typ = 'NUMBER'
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
59 elif typ == 'REAL':
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
60 val = float(val)
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
61 elif typ == 'STRING':
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
62 val = val[1:-1]
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
63 yield Token(typ, val, line, mo.start()-line_start)
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
64 pos = mo.end()
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
65 mo = gettok(s, pos)
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
66 if pos != len(s):
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
67 col = pos - line_start
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
68 raise CompilerException('Unexpected character {0}'.format(s[pos]), line, col)
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
69 yield Token('END', '', line, 0)
92df07bc2081 Initial import of compiler
windel
parents:
diff changeset
70