Mercurial > lcfOS

comparison python/c3/lexer.py @ 163:8104fc8b5e90

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added visitor to c3
author Windel Bouwman
date Mon, 18 Mar 2013 20:13:57 +0100
parents b73bc14a3aa3
children 6b2bec5653f1
comparison
equal deleted inserted replaced
162:d8c735dc31f9 163:8104fc8b5e90
9 # Token is used in the lexical analyzer: 9 # Token is used in the lexical analyzer:
10 Token = collections.namedtuple('Token', 'typ val loc') 10 Token = collections.namedtuple('Token', 'typ val loc')
11 11
12 keywords = ['and', 'or', 'not','true', 'false', \ 12 keywords = ['and', 'or', 'not','true', 'false', \
13 'else', 'if', 'while', 'return', \ 13 'else', 'if', 'while', 'return', \
14 'function', 'var', 'type', \ 14 'function', 'var', 'type', 'const', \
15 'import', 'package' ] 15 'import', 'package' ]
16 16
17 def tokenize(s): 17 def tokenize(s):
18 """ 18 """
19 Tokenizer, generates an iterator that 19 Tokenizer, generates an iterator that
60 typ = 'NUMBER' 60 typ = 'NUMBER'
61 elif typ == 'REAL': 61 elif typ == 'REAL':
62 val = float(val) 62 val = float(val)
63 elif typ == 'STRING': 63 elif typ == 'STRING':
64 val = val[1:-1] 64 val = val[1:-1]
65 loc = SourceLocation(line, mo.start()-line_start) 65 loc = SourceLocation(line, mo.start()-line_start, mo.end() - mo.start())
66 yield Token(typ, val, loc) 66 yield Token(typ, val, loc)
67 pos = mo.end() 67 pos = mo.end()
68 mo = gettok(s, pos) 68 mo = gettok(s, pos)
69 if pos != len(s): 69 if pos != len(s):
70 col = pos - line_start 70 col = pos - line_start