annotate python/ppci/c3/lexer.py @ 300:158068af716c

yafm
author Windel Bouwman
date Tue, 03 Dec 2013 18:00:22 +0100
parents python/c3/lexer.py@6aa721e7b10b
children 0615b5308710
rev   line source
288
a747a45dcd78 Various styling work
Windel Bouwman
parents: 287
diff changeset
1 import collections
a747a45dcd78 Various styling work
Windel Bouwman
parents: 287
diff changeset
2 import re
152
b73bc14a3aa3 Light coupling ide and c3 frontend
Windel Bouwman
parents: 149
diff changeset
3
191
6b2bec5653f1 Added assembler testset
Windel Bouwman
parents: 163
diff changeset
4 from ppci import CompilerError, SourceLocation, Token
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
5
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
6 """
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
7 Lexical analyzer part. Splits the input character stream into tokens.
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
8 """
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
9
288
a747a45dcd78 Various styling work
Windel Bouwman
parents: 287
diff changeset
10 keywords = ['and', 'or', 'not', 'true', 'false',
300
Windel Bouwman
parents: 293
diff changeset
11 'else', 'if', 'while', 'return',
Windel Bouwman
parents: 293
diff changeset
12 'function', 'var', 'type', 'const',
Windel Bouwman
parents: 293
diff changeset
13 'struct', 'cast',
Windel Bouwman
parents: 293
diff changeset
14 'import', 'module']
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
15
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
16
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
17 class Lexer:
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
18 def __init__(self, diag):
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
19 self.diag = diag
148
e5263f74b287 Added c3 language frontend initial parser
Windel Bouwman
parents:
diff changeset
20
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
21 def tokenize(self, input_file):
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
22 """
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
23 Tokenizer, generates an iterator that
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
24 returns tokens!
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
25
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
26 Input is a file like object.
287
1c7c1e619be8 File movage
Windel Bouwman
parents: 284
diff changeset
27
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
28 This GREAT example was taken from python re doc page!
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
29 """
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
30 filename = input_file.name if hasattr(input_file, 'name') else ''
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
31 s = input_file.read()
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
32 input_file.close()
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
33 self.diag.addSource(filename, s)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
34 tok_spec = [
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
35 ('REAL', r'\d+\.\d+'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
36 ('HEXNUMBER', r'0x[\da-fA-F]+'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
37 ('NUMBER', r'\d+'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
38 ('ID', r'[A-Za-z][A-Za-z\d_]*'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
39 ('NEWLINE', r'\n'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
40 ('SKIP', r'[ \t]'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
41 ('COMMENTS', r'//.*'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
42 ('LONGCOMMENTBEGIN', r'\/\*'),
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
43 ('LONGCOMMENTEND', r'\*\/'),
300
Windel Bouwman
parents: 293
diff changeset
44 ('LEESTEKEN', r'==|->|<<|>>|!=|[\.,=:;\-+*\[\]/\(\)]|>=|<=|<>|>|<|{|}|&|\^|\|'),
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
45 ('STRING', r"'.*?'")
300
Windel Bouwman
parents: 293
diff changeset
46 ]
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
47 tok_re = '|'.join('(?P<%s>%s)' % pair for pair in tok_spec)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
48 gettok = re.compile(tok_re).match
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
49 line = 1
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
50 pos = line_start = 0
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
51 mo = gettok(s)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
52 incomment = False
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
53 while mo is not None:
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
54 typ = mo.lastgroup
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
55 val = mo.group(typ)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
56 if typ == 'NEWLINE':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
57 line_start = pos
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
58 line += 1
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
59 elif typ == 'COMMENTS':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
60 pass
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
61 elif typ == 'LONGCOMMENTBEGIN':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
62 incomment = True
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
63 elif typ == 'LONGCOMMENTEND':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
64 incomment = False
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
65 elif typ == 'SKIP':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
66 pass
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
67 elif incomment:
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
68 pass # Wait until we are not in a comment section
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
69 else:
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
70 if typ == 'ID':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
71 if val in keywords:
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
72 typ = val
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
73 elif typ == 'LEESTEKEN':
288
a747a45dcd78 Various styling work
Windel Bouwman
parents: 287
diff changeset
74 typ = val
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
75 elif typ == 'NUMBER':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
76 val = int(val)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
77 elif typ == 'HEXNUMBER':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
78 val = int(val[2:], 16)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
79 typ = 'NUMBER'
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
80 elif typ == 'REAL':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
81 val = float(val)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
82 elif typ == 'STRING':
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
83 val = val[1:-1]
300
Windel Bouwman
parents: 293
diff changeset
84 loc = SourceLocation(filename, line, mo.start() - line_start,
Windel Bouwman
parents: 293
diff changeset
85 mo.end() - mo.start())
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
86 yield Token(typ, val, loc)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
87 pos = mo.end()
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
88 mo = gettok(s, pos)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
89 if pos != len(s):
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
90 col = pos - line_start
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
91 loc = SourceLocation(filename, line, col, 1)
300
Windel Bouwman
parents: 293
diff changeset
92 raise CompilerError('Unexpected: "{0}"'.format(s[pos]), loc)
293
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
93 loc = SourceLocation(filename, line, 0, 0)
6aa721e7b10b Try to improve build sequence
Windel Bouwman
parents: 288
diff changeset
94 yield Token('END', '', loc)