Mercurial > lcfOS

comparison python/ppci/c3/lexer.py @ 305:0615b5308710

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Updated docs
author Windel Bouwman
date Fri, 06 Dec 2013 13:50:38 +0100
parents 158068af716c
children b145f8e6050b
comparison
equal deleted inserted replaced
304:fa99f36fabb5 305:0615b5308710
6 """ 6 """
7 Lexical analyzer part. Splits the input character stream into tokens. 7 Lexical analyzer part. Splits the input character stream into tokens.
8 """ 8 """
9 9
10 keywords = ['and', 'or', 'not', 'true', 'false', 10 keywords = ['and', 'or', 'not', 'true', 'false',
11 'else', 'if', 'while', 'return', 11 'else', 'if', 'while', 'return',
12 'function', 'var', 'type', 'const', 12 'function', 'var', 'type', 'const',
13 'struct', 'cast', 13 'struct', 'cast',
14 'import', 'module'] 14 'import', 'module']
15 15
16 16
17 class Lexer: 17 class Lexer:
18 """ Generates a sequence of token from an input stream """
18 def __init__(self, diag): 19 def __init__(self, diag):
19 self.diag = diag 20 self.diag = diag
20 21
21 def tokenize(self, input_file): 22 def tokenize(self, input_file):
22 """ 23 """
63 elif typ == 'LONGCOMMENTEND': 64 elif typ == 'LONGCOMMENTEND':
64 incomment = False 65 incomment = False
65 elif typ == 'SKIP': 66 elif typ == 'SKIP':
66 pass 67 pass
67 elif incomment: 68 elif incomment:
68 pass # Wait until we are not in a comment section 69 pass # Wait until we are not in a comment section
69 else: 70 else:
70 if typ == 'ID': 71 if typ == 'ID':
71 if val in keywords: 72 if val in keywords:
72 typ = val 73 typ = val
73 elif typ == 'LEESTEKEN': 74 elif typ == 'LEESTEKEN':
80 elif typ == 'REAL': 81 elif typ == 'REAL':
81 val = float(val) 82 val = float(val)
82 elif typ == 'STRING': 83 elif typ == 'STRING':
83 val = val[1:-1] 84 val = val[1:-1]
84 loc = SourceLocation(filename, line, mo.start() - line_start, 85 loc = SourceLocation(filename, line, mo.start() - line_start,
85 mo.end() - mo.start()) 86 mo.end() - mo.start())
86 yield Token(typ, val, loc) 87 yield Token(typ, val, loc)
87 pos = mo.end() 88 pos = mo.end()
88 mo = gettok(s, pos) 89 mo = gettok(s, pos)
89 if pos != len(s): 90 if pos != len(s):
90 col = pos - line_start 91 col = pos - line_start