Mercurial > lcfOS

comparison python/pyburg.py @ 396:fb3c1f029b30

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added baselexer into c3 lexer
author Windel Bouwman
date Tue, 27 May 2014 22:19:32 +0200
parents 0c44e494ef58
children
comparison
equal deleted inserted replaced
395:3b0c495e3008 396:fb3c1f029b30
57 import sys 57 import sys
58 import os 58 import os
59 import io 59 import io
60 import types 60 import types
61 import argparse 61 import argparse
62 from ppci import Token 62 from ppci import Token, SourceLocation
63 from pyyacc import ParserException 63 from pyyacc import ParserException
64 import yacc 64 import yacc
65 import baselex 65 import baselex
66 from tree import Tree 66 from tree import Tree
67 67
85 def tokenize(self, txt): 85 def tokenize(self, txt):
86 lines = txt.split('\n') 86 lines = txt.split('\n')
87 header_lines = [] 87 header_lines = []
88 section = 0 88 section = 0
89 for line in lines: 89 for line in lines:
90 loc = SourceLocation(self.filename, 0, 0, 0)
90 line = line.strip() 91 line = line.strip()
91 if not line: 92 if not line:
92 continue # Skip empty lines 93 continue # Skip empty lines
93 elif line == '%%': 94 elif line == '%%':
94 section += 1 95 section += 1
95 if section == 1: 96 if section == 1:
96 yield Token('header', header_lines) 97 yield Token('header', header_lines, loc)
97 yield Token('%%', '%%') 98 yield Token('%%', '%%', loc)
98 else: 99 else:
99 if section == 0: 100 if section == 0:
100 header_lines.append(line) 101 header_lines.append(line)
101 else: 102 else:
102 # we could use yield from below, but python 3.2 does not work then: 103 # we could use yield from below, but python 3.2 does not work then: