annotate python/ppci/c3/lexer.py @ 311:ff665880a6b0

Added testcase for kernel and userspace
author Windel Bouwman
date Mon, 16 Dec 2013 12:49:24 +0100
parents b145f8e6050b
children 084cccaa5deb
rev   line source
288
a747a45dcd78 Various styling work
Windel Bouwman
parents: 287
diff changeset
1 import collections
a747a45dcd78 Various styling work
Windel Bouwman
parents: 287
diff changeset
2 import re
152
b73bc14a3aa3 Light coupling ide and c3 frontend
Windel Bouwman
parents: 149
diff changeset
3
191
6b2bec5653f1 Added assembler testset
Windel Bouwman
parents: 163
diff changeset
4 from ppci import CompilerError, SourceLocation, Token
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
5
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
6 """
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
7 Lexical analyzer part. Splits the input character stream into tokens.
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
8 """
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
9
288
a747a45dcd78 Various styling work
Windel Bouwman
parents: 287
diff changeset
10 keywords = ['and', 'or', 'not', 'true', 'false',
305
0615b5308710 Updated docs
Windel Bouwman
parents: 300
diff changeset
11 'else', 'if', 'while', 'return',
0615b5308710 Updated docs
Windel Bouwman
parents: 300
diff changeset
12 'function', 'var', 'type', 'const',
0615b5308710 Updated docs
Windel Bouwman
parents: 300
diff changeset
13 'struct', 'cast',
0615b5308710 Updated docs
Windel Bouwman
parents: 300
diff changeset
14 'import', 'module']
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
15
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
16
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
17 class Lexer:
305
0615b5308710 Updated docs
Windel Bouwman
parents: 300
diff changeset
18 """ Generates a sequence of token from an input stream """
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
19 def __init__(self, diag):
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
20 self.diag = diag
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
21
306
b145f8e6050b Start on c3 rewrite
Windel Bouwman
parents: 305
diff changeset
22 def lex(self, source):
b145f8e6050b Start on c3 rewrite
Windel Bouwman
parents: 305
diff changeset
23 return self.tokenize(source)
b145f8e6050b Start on c3 rewrite
Windel Bouwman
parents: 305
diff changeset
24
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
25 def tokenize(self, input_file):
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
26 """
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
27 Tokenizer, generates an iterator that
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
28 returns tokens!
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
29
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
30 Input is a file like object.
287
1c7c1e619be8 File movage
Windel Bouwman
parents: 284
diff changeset
31
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
32 This GREAT example was taken from python re doc page!
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
33 """
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
34 filename = input_file.name if hasattr(input_file, 'name') else ''
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
35 s = input_file.read()
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
36 input_file.close()
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
37 self.diag.addSource(filename, s)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
38 tok_spec = [
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
39 ('REAL', r'\d+\.\d+'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
40 ('HEXNUMBER', r'0x[\da-fA-F]+'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
41 ('NUMBER', r'\d+'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
42 ('ID', r'[A-Za-z][A-Za-z\d_]*'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
43 ('NEWLINE', r'\n'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
44 ('SKIP', r'[ \t]'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
45 ('COMMENTS', r'//.*'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
46 ('LONGCOMMENTBEGIN', r'\/\*'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
47 ('LONGCOMMENTEND', r'\*\/'),
300
Windel Bouwman
parents: 293
diff changeset
48 ('LEESTEKEN', r'==|->|<<|>>|!=|[\.,=:;\-+*\[\]/\(\)]|>=|<=|<>|>|<|{|}|&|\^|\|'),
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
49 ('STRING', r"'.*?'")
300
Windel Bouwman
parents: 293
diff changeset
50 ]
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
51 tok_re = '|'.join('(?P<%s>%s)' % pair for pair in tok_spec)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
52 gettok = re.compile(tok_re).match
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
53 line = 1
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
54 pos = line_start = 0
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
55 mo = gettok(s)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
56 incomment = False
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
57 while mo is not None:
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
58 typ = mo.lastgroup
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
59 val = mo.group(typ)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
60 if typ == 'NEWLINE':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
61 line_start = pos
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
62 line += 1
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
63 elif typ == 'COMMENTS':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
64 pass
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
65 elif typ == 'LONGCOMMENTBEGIN':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
66 incomment = True
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
67 elif typ == 'LONGCOMMENTEND':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
68 incomment = False
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
69 elif typ == 'SKIP':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
70 pass
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
71 elif incomment:
305
0615b5308710 Updated docs
Windel Bouwman
parents: 300
diff changeset
72 pass # Wait until we are not in a comment section
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
73 else:
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
74 if typ == 'ID':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
75 if val in keywords:
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
76 typ = val
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
77 elif typ == 'LEESTEKEN':
288
a747a45dcd78 Various styling work
Windel Bouwman
parents: 287
diff changeset
78 typ = val
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
79 elif typ == 'NUMBER':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
80 val = int(val)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
81 elif typ == 'HEXNUMBER':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
82 val = int(val[2:], 16)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
83 typ = 'NUMBER'
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
84 elif typ == 'REAL':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
85 val = float(val)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
86 elif typ == 'STRING':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
87 val = val[1:-1]
300
Windel Bouwman
parents: 293
diff changeset
88 loc = SourceLocation(filename, line, mo.start() - line_start,
305
0615b5308710 Updated docs
Windel Bouwman
parents: 300
diff changeset
89 mo.end() - mo.start())
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
90 yield Token(typ, val, loc)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
91 pos = mo.end()
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
92 mo = gettok(s, pos)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
93 if pos != len(s):
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
94 col = pos - line_start
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
95 loc = SourceLocation(filename, line, col, 1)
300
Windel Bouwman
parents: 293
diff changeset
96 raise CompilerError('Unexpected: "{0}"'.format(s[pos]), loc)
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
97 loc = SourceLocation(filename, line, 0, 0)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
98 yield Token('END', '', loc)