annotate python/c3/lexer.py @ 281:4496cae24d7f

Improved logview
author Windel Bouwman
date Sat, 02 Nov 2013 11:11:40 +0100
parents e621e3ba78d2
children 05184b95fa16
rev   line source
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
1 import collections, re
152
b73bc14a3aa3 Light coupling ide and c3 frontend
Windel Bouwman
parents: 149
diff changeset
2
191
6b2bec5653f1 Added assembler testset
Windel Bouwman
parents: 163
diff changeset
3 from ppci import CompilerError, SourceLocation, Token
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
4
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
5 """
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
6 Lexical analyzer part. Splits the input character stream into tokens.
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
7 """
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
8
213
003c8a976fff Merge of semantics and parser again ..
Windel Bouwman
parents: 205
diff changeset
9 keywords = ['and', 'or', 'not', 'true', 'false', \
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
10 'else', 'if', 'while', 'return', \
163
8104fc8b5e90 Added visitor to c3
Windel Bouwman
parents: 152
diff changeset
11 'function', 'var', 'type', 'const', \
221
848c4b15fd0b pointers
Windel Bouwman
parents: 213
diff changeset
12 'struct', 'cast', \
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
13 'import', 'package' ]
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
14
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
15 def tokenize(s):
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
16 """
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
17 Tokenizer, generates an iterator that
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
18 returns tokens!
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
19
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
20 This GREAT example was taken from python re doc page!
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
21 """
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
22 tok_spec = [
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
23 ('REAL', r'\d+\.\d+'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
24 ('HEXNUMBER', r'0x[\da-fA-F]+'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
25 ('NUMBER', r'\d+'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
26 ('ID', r'[A-Za-z][A-Za-z\d_]*'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
27 ('NEWLINE', r'\n'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
28 ('SKIP', r'[ \t]'),
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
29 ('COMMENTS', r'//.*'),
204
de3a68f677a5 Added long comment to c3 parser
Windel Bouwman
parents: 194
diff changeset
30 ('LONGCOMMENTBEGIN', r'\/\*'),
de3a68f677a5 Added long comment to c3 parser
Windel Bouwman
parents: 194
diff changeset
31 ('LONGCOMMENTEND', r'\*\/'),
232
e621e3ba78d2 Added left shift instruction
Windel Bouwman
parents: 225
diff changeset
32 ('LEESTEKEN', r'==|->|<<|>>|[\.,=:;\-+*\[\]/\(\)]|>=|<=|<>|>|<|{|}|&|\^|\|'),
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
33 ('STRING', r"'.*?'")
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
34 ]
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
35 tok_re = '|'.join('(?P<%s>%s)' % pair for pair in tok_spec)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
36 gettok = re.compile(tok_re).match
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
37 line = 1
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
38 pos = line_start = 0
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
39 mo = gettok(s)
204
de3a68f677a5 Added long comment to c3 parser
Windel Bouwman
parents: 194
diff changeset
40 incomment = False
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
41 while mo is not None:
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
42 typ = mo.lastgroup
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
43 val = mo.group(typ)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
44 if typ == 'NEWLINE':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
45 line_start = pos
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
46 line += 1
205
d77cb5962cc5 Added some handcoded arm code generation
Windel Bouwman
parents: 204
diff changeset
47 elif typ == 'COMMENTS':
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
48 pass
204
de3a68f677a5 Added long comment to c3 parser
Windel Bouwman
parents: 194
diff changeset
49 elif typ == 'LONGCOMMENTBEGIN':
de3a68f677a5 Added long comment to c3 parser
Windel Bouwman
parents: 194
diff changeset
50 incomment = True
de3a68f677a5 Added long comment to c3 parser
Windel Bouwman
parents: 194
diff changeset
51 elif typ == 'LONGCOMMENTEND':
de3a68f677a5 Added long comment to c3 parser
Windel Bouwman
parents: 194
diff changeset
52 incomment = False
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
53 elif typ == 'SKIP':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
54 pass
204
de3a68f677a5 Added long comment to c3 parser
Windel Bouwman
parents: 194
diff changeset
55 elif incomment:
de3a68f677a5 Added long comment to c3 parser
Windel Bouwman
parents: 194
diff changeset
56 pass # Wait until we are not in a comment section
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
57 else:
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
58 if typ == 'ID':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
59 if val in keywords:
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
60 typ = val
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
61 elif typ == 'LEESTEKEN':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
62 typ = val
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
63 elif typ == 'NUMBER':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
64 val = int(val)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
65 elif typ == 'HEXNUMBER':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
66 val = int(val[2:], 16)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
67 typ = 'NUMBER'
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
68 elif typ == 'REAL':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
69 val = float(val)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
70 elif typ == 'STRING':
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
71 val = val[1:-1]
163
8104fc8b5e90 Added visitor to c3
Windel Bouwman
parents: 152
diff changeset
72 loc = SourceLocation(line, mo.start()-line_start, mo.end() - mo.start())
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
73 yield Token(typ, val, loc)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
74 pos = mo.end()
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
75 mo = gettok(s, pos)
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
76 if pos != len(s):
205
d77cb5962cc5 Added some handcoded arm code generation
Windel Bouwman
parents: 204
diff changeset
77 col = pos - line_start
d77cb5962cc5 Added some handcoded arm code generation
Windel Bouwman
parents: 204
diff changeset
78 loc = SourceLocation(line, col, 1)
d77cb5962cc5 Added some handcoded arm code generation
Windel Bouwman
parents: 204
diff changeset
79 raise CompilerError('Unexpected character "{0}"'.format(s[pos]), loc)
194
b01429a5d695 Fixed test
Windel Bouwman
parents: 191
diff changeset
80 loc = SourceLocation(line, 0, 0)
b01429a5d695 Fixed test
Windel Bouwman
parents: 191
diff changeset
81 yield Token('END', '', loc)
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
82