annotate python/c3/lexer.py @ 163:8104fc8b5e90

Added visitor to c3
author Windel Bouwman
date Mon, 18 Mar 2013 20:13:57 +0100
parents b73bc14a3aa3
children 6b2bec5653f1
rev   line source
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
1 import collections, re
152
b73bc14a3aa3 Light coupling ide and c3 frontend
Windel Bouwman
parents: 149
diff changeset
2
b73bc14a3aa3 Light coupling ide and c3 frontend
Windel Bouwman
parents: 149
diff changeset
3 from ppci import CompilerError, SourceLocation
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
4
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
5 """
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
6 Lexical analyzer part. Splits the input character stream into tokens.
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
7 """
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
8
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
9 # Token is used in the lexical analyzer:
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
10 Token = collections.namedtuple('Token', 'typ val loc')
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
11
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
12 keywords = ['and', 'or', 'not','true', 'false', \
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
13 'else', 'if', 'while', 'return', \
163
8104fc8b5e90 Added visitor to c3
Windel Bouwman
parents: 152
diff changeset
14 'function', 'var', 'type', 'const', \
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
15 'import', 'package' ]
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
16
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
17 def tokenize(s):
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
18 """
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
19 Tokenizer, generates an iterator that
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
20 returns tokens!
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
21
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
22 This GREAT example was taken from python re doc page!
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
23 """
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
24 tok_spec = [
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
25 ('REAL', r'\d+\.\d+'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
26 ('HEXNUMBER', r'0x[\da-fA-F]+'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
27 ('NUMBER', r'\d+'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
28 ('ID', r'[A-Za-z][A-Za-z\d_]*'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
29 ('NEWLINE', r'\n'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
30 ('SKIP', r'[ \t]'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
31 ('COMMENTS', r'//.*'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
32 ('LEESTEKEN', r'==|[\.,=:;\-+*\[\]/\(\)]|>=|<=|<>|>|<|{|}'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
33 ('STRING', r"'.*?'")
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
34 ]
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
35 tok_re = '|'.join('(?P<%s>%s)' % pair for pair in tok_spec)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
36 gettok = re.compile(tok_re).match
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
37 line = 1
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
38 pos = line_start = 0
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
39 mo = gettok(s)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
40 while mo is not None:
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
41 typ = mo.lastgroup
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
42 val = mo.group(typ)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
43 if typ == 'NEWLINE':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
44 line_start = pos
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
45 line += 1
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
46 elif typ == 'COMMENTS':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
47 pass
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
48 elif typ == 'SKIP':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
49 pass
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
50 else:
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
51 if typ == 'ID':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
52 if val in keywords:
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
53 typ = val
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
54 elif typ == 'LEESTEKEN':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
55 typ = val
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
56 elif typ == 'NUMBER':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
57 val = int(val)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
58 elif typ == 'HEXNUMBER':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
59 val = int(val[2:], 16)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
60 typ = 'NUMBER'
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
61 elif typ == 'REAL':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
62 val = float(val)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
63 elif typ == 'STRING':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
64 val = val[1:-1]
163
8104fc8b5e90 Added visitor to c3
Windel Bouwman
parents: 152
diff changeset
65 loc = SourceLocation(line, mo.start()-line_start, mo.end() - mo.start())
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
66 yield Token(typ, val, loc)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
67 pos = mo.end()
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
68 mo = gettok(s, pos)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
69 if pos != len(s):
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
70 col = pos - line_start
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
71 pos = line
152
b73bc14a3aa3 Light coupling ide and c3 frontend
Windel Bouwman
parents: 149
diff changeset
72 raise CompilerError('Unexpected character {0}'.format(s[pos]), pos)
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
73 yield Token('END', '', line)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
74