|
288
|
1 import collections
|
|
|
2 import re
|
|
152
|
3
|
|
191
|
4 from ppci import CompilerError, SourceLocation, Token
|
|
148
|
5
|
|
|
6 """
|
|
|
7 Lexical analyzer part. Splits the input character stream into tokens.
|
|
|
8 """
|
|
|
9
|
|
288
|
10 keywords = ['and', 'or', 'not', 'true', 'false',
|
|
305
|
11 'else', 'if', 'while', 'return',
|
|
|
12 'function', 'var', 'type', 'const',
|
|
|
13 'struct', 'cast',
|
|
|
14 'import', 'module']
|
|
148
|
15
|
|
293
|
16
|
|
|
17 class Lexer:
|
|
305
|
18 """ Generates a sequence of token from an input stream """
|
|
293
|
19 def __init__(self, diag):
|
|
|
20 self.diag = diag
|
|
148
|
21
|
|
306
|
22 def lex(self, source):
|
|
|
23 return self.tokenize(source)
|
|
|
24
|
|
293
|
25 def tokenize(self, input_file):
|
|
|
26 """
|
|
|
27 Tokenizer, generates an iterator that
|
|
|
28 returns tokens!
|
|
|
29
|
|
|
30 Input is a file like object.
|
|
287
|
31
|
|
293
|
32 This GREAT example was taken from python re doc page!
|
|
|
33 """
|
|
|
34 filename = input_file.name if hasattr(input_file, 'name') else ''
|
|
|
35 s = input_file.read()
|
|
|
36 input_file.close()
|
|
|
37 self.diag.addSource(filename, s)
|
|
|
38 tok_spec = [
|
|
|
39 ('REAL', r'\d+\.\d+'),
|
|
|
40 ('HEXNUMBER', r'0x[\da-fA-F]+'),
|
|
|
41 ('NUMBER', r'\d+'),
|
|
|
42 ('ID', r'[A-Za-z][A-Za-z\d_]*'),
|
|
|
43 ('NEWLINE', r'\n'),
|
|
|
44 ('SKIP', r'[ \t]'),
|
|
|
45 ('COMMENTS', r'//.*'),
|
|
|
46 ('LONGCOMMENTBEGIN', r'\/\*'),
|
|
|
47 ('LONGCOMMENTEND', r'\*\/'),
|
|
300
|
48 ('LEESTEKEN', r'==|->|<<|>>|!=|[\.,=:;\-+*\[\]/\(\)]|>=|<=|<>|>|<|{|}|&|\^|\|'),
|
|
293
|
49 ('STRING', r"'.*?'")
|
|
300
|
50 ]
|
|
293
|
51 tok_re = '|'.join('(?P<%s>%s)' % pair for pair in tok_spec)
|
|
|
52 gettok = re.compile(tok_re).match
|
|
|
53 line = 1
|
|
|
54 pos = line_start = 0
|
|
|
55 mo = gettok(s)
|
|
|
56 incomment = False
|
|
|
57 while mo is not None:
|
|
|
58 typ = mo.lastgroup
|
|
|
59 val = mo.group(typ)
|
|
|
60 if typ == 'NEWLINE':
|
|
|
61 line_start = pos
|
|
|
62 line += 1
|
|
|
63 elif typ == 'COMMENTS':
|
|
|
64 pass
|
|
|
65 elif typ == 'LONGCOMMENTBEGIN':
|
|
|
66 incomment = True
|
|
|
67 elif typ == 'LONGCOMMENTEND':
|
|
|
68 incomment = False
|
|
|
69 elif typ == 'SKIP':
|
|
|
70 pass
|
|
|
71 elif incomment:
|
|
305
|
72 pass # Wait until we are not in a comment section
|
|
293
|
73 else:
|
|
|
74 if typ == 'ID':
|
|
|
75 if val in keywords:
|
|
|
76 typ = val
|
|
|
77 elif typ == 'LEESTEKEN':
|
|
288
|
78 typ = val
|
|
293
|
79 elif typ == 'NUMBER':
|
|
|
80 val = int(val)
|
|
|
81 elif typ == 'HEXNUMBER':
|
|
|
82 val = int(val[2:], 16)
|
|
|
83 typ = 'NUMBER'
|
|
|
84 elif typ == 'REAL':
|
|
|
85 val = float(val)
|
|
|
86 elif typ == 'STRING':
|
|
|
87 val = val[1:-1]
|
|
300
|
88 loc = SourceLocation(filename, line, mo.start() - line_start,
|
|
305
|
89 mo.end() - mo.start())
|
|
293
|
90 yield Token(typ, val, loc)
|
|
|
91 pos = mo.end()
|
|
|
92 mo = gettok(s, pos)
|
|
|
93 if pos != len(s):
|
|
|
94 col = pos - line_start
|
|
|
95 loc = SourceLocation(filename, line, col, 1)
|
|
300
|
96 raise CompilerError('Unexpected: "{0}"'.format(s[pos]), loc)
|
|
293
|
97 loc = SourceLocation(filename, line, 0, 0)
|
|
|
98 yield Token('END', '', loc)
|
