annotate python/yacc.py @ 368:d2ddfe134c48

Remove yield from for python < 3.3
author Windel Bouwman
date Fri, 21 Mar 2014 11:21:50 +0100
parents 86b02c98a717
children 173e20a47fda
rev   line source
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
1 #!/usr/bin/python
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
2
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
3 """
334
6f4753202b9a Added more recipes
Windel Bouwman
parents: 323
diff changeset
4 Parser generator utility. This script can generate a python script from a
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
5 grammar description.
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
6
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
7 Invoke the script on a grammar specification file:
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
8
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
9 .. code::
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
10
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
11 $ ./yacc.py test.x -o test_parser.py
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
12
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
13 And use the generated parser by deriving a user class:
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
14
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
15
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
16 .. code::
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
17
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
18 import test_parser
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
19 class MyParser(test_parser.Parser):
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
20 pass
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
21 p = MyParser()
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
22 p.parse()
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
23
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
24
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
25 Alternatively you can load the parser on the fly:
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
26
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
27 .. code::
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
28
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
29 import yacc
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
30 parser_mod = yacc.load_as_module('mygrammar.x')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
31 class MyParser(parser_mod.Parser):
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
32 pass
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
33 p = MyParser()
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
34 p.parse()
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
35
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
36 """
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
37
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
38 import argparse
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
39 import re
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
40 import sys
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
41 import datetime
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
42 import types
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
43 import io
323
e9fe6988497c Used burg for generating expressions
Windel Bouwman
parents: 322
diff changeset
44 import logging
334
6f4753202b9a Added more recipes
Windel Bouwman
parents: 323
diff changeset
45 from pyyacc import Grammar
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
46
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
47
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
48 class XaccLexer:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
49 def __init__(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
50 pass
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
51
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
52 def feed(self, txt):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
53 # Create a regular expression for the lexing part:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
54 tok_spec = [
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
55 ('ID', r'[A-Za-z][A-Za-z\d_]*'),
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
56 ('STRING', r"'[^']*'"),
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
57 ('BRACEDCODE', r"\{[^\}]*\}"),
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
58 ('OTHER', r'[:;\|]'),
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
59 ('SKIP', r'[ ]')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
60 ]
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
61 tok_re = '|'.join('(?P<%s>%s)' % pair for pair in tok_spec)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
62 gettok = re.compile(tok_re).match
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
63
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
64 lines = txt.split('\n')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
65
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
66 def tokenize_line(line):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
67 """ Generator that splits up a line into tokens """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
68 mo = gettok(line)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
69 pos = 0
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
70 while mo:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
71 typ = mo.lastgroup
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
72 val = mo.group(typ)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
73 if typ == 'ID':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
74 yield (typ, val)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
75 elif typ == 'STRING':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
76 typ = 'ID'
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
77 yield (typ, val[1:-1])
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
78 elif typ == 'OTHER':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
79 typ = val
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
80 yield (typ, val)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
81 elif typ == 'BRACEDCODE':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
82 yield (typ, val)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
83 elif typ == 'SKIP':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
84 pass
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
85 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
86 raise NotImplementedError(str(typ))
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
87 pos = mo.end()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
88 mo = gettok(line, pos)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
89 if len(line) != pos:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
90 raise ParseError('Lex fault at {}'.format(line))
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
91
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
92 def tokenize():
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
93 section = 0
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
94 for line in lines:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
95 line = line.strip()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
96 if not line:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
97 continue # Skip empty lines
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
98 if line == '%%':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
99 section += 1
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
100 yield('%%', '%%')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
101 continue
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
102 if section == 0:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
103 if line.startswith('%tokens'):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
104 yield('%tokens', '%tokens')
368
d2ddfe134c48 Remove yield from for python < 3.3
Windel Bouwman
parents: 342
diff changeset
105 for tk in tokenize_line(line[7:]):
d2ddfe134c48 Remove yield from for python < 3.3
Windel Bouwman
parents: 342
diff changeset
106 yield tk
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
107 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
108 yield ('HEADER', line)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
109 elif section == 1:
368
d2ddfe134c48 Remove yield from for python < 3.3
Windel Bouwman
parents: 342
diff changeset
110 for tk in tokenize_line(line):
d2ddfe134c48 Remove yield from for python < 3.3
Windel Bouwman
parents: 342
diff changeset
111 yield tk
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
112 yield ('eof', 'eof')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
113 self.tokens = tokenize()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
114 self.token = self.tokens.__next__()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
115
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
116 def next_token(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
117 t = self.token
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
118 if t[0] != 'eof':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
119 self.token = self.tokens.__next__()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
120 return t
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
121
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
122
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
123 class ParseError(Exception):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
124 pass
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
125
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
126
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
127 class XaccParser:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
128 """ Implements a recursive descent parser to parse grammar rules.
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
129 We could have made an generated parser, but that would yield a chicken
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
130 egg issue.
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
131 """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
132 def __init__(self, lexer):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
133 self.lexer = lexer
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
134
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
135 @property
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
136 def Peak(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
137 """ Sneak peak to the next token in line """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
138 return self.lexer.token[0]
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
139
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
140 def next_token(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
141 """ Take the next token """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
142 return self.lexer.next_token()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
143
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
144 def consume(self, typ):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
145 """ Eat next token of type typ or raise an exception """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
146 if self.Peak == typ:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
147 return self.next_token()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
148 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
149 raise ParseError('Expected {}, but got {}'.format(typ, self.Peak))
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
150
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
151 def has_consumed(self, typ):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
152 """ Consume typ if possible and return true if so """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
153 if self.Peak == typ:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
154 self.consume(typ)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
155 return True
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
156 return False
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
157
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
158 def parse_grammar(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
159 """ Entry parse function into recursive descent parser """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
160 # parse header
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
161 headers = []
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
162 terminals = []
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
163 while self.Peak in ['HEADER', '%tokens']:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
164 if self.Peak == '%tokens':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
165 self.consume('%tokens')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
166 while self.Peak == 'ID':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
167 terminals.append(self.consume('ID')[1])
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
168 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
169 headers.append(self.consume('HEADER')[1])
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
170 self.consume('%%')
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
171 self.headers = headers
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
172 self.grammar = Grammar(terminals)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
173 while self.Peak != 'eof':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
174 self.parse_rule()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
175 return self.grammar
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
176
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
177 def parse_symbol(self):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
178 return self.consume('ID')[1]
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
179
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
180 def parse_rhs(self):
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
181 """ Parse the right hand side of a rule definition """
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
182 symbols = []
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
183 while self.Peak not in [';', 'BRACEDCODE', '|']:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
184 symbols.append(self.parse_symbol())
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
185 if self.Peak == 'BRACEDCODE':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
186 action = self.consume('BRACEDCODE')[1]
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
187 action = action[1:-1].strip()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
188 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
189 action = None
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
190 return symbols, action
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
191
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
192 def parse_rule(self):
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
193 """ Parse a rule definition """
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
194 p = self.parse_symbol()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
195 self.consume(':')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
196 symbols, action = self.parse_rhs()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
197 self.grammar.add_production(p, symbols, action)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
198 while self.has_consumed('|'):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
199 symbols, action = self.parse_rhs()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
200 self.grammar.add_production(p, symbols, action)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
201 self.consume(';')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
202
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
203
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
204 class XaccGenerator:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
205 """ Generator that writes generated parser to file """
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
206 def __init__(self):
323
e9fe6988497c Used burg for generating expressions
Windel Bouwman
parents: 322
diff changeset
207 self.logger = logging.getLogger('yacc')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
208
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
209 def generate(self, grammar, headers, output_file):
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
210 self.output_file = output_file
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
211 self.grammar = grammar
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
212 self.headers = headers
323
e9fe6988497c Used burg for generating expressions
Windel Bouwman
parents: 322
diff changeset
213 self.logger.info('Generating parser for grammar {}'.format(grammar))
340
c7cc54c0dfdf Test featurebranch
Windel Bouwman
parents: 334
diff changeset
214 self.action_table, self.goto_table = grammar.generate_tables()
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
215 self.generate_python_script()
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
216
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
217 def print(self, *args):
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
218 """ Print helper function that prints to output file """
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
219 print(*args, file=self.output_file)
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
220
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
221 def generate_python_script(self):
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
222 """ Generate python script with the parser table """
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
223 self.print('#!/usr/bin/python')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
224 stamp = datetime.datetime.now().ctime()
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
225 self.print('""" Automatically generated by xacc on {} """'.format(stamp))
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
226 self.print('from pyyacc import LRParser, Reduce, Shift, Accept, Production, Grammar')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
227 self.print('from ppci import Token')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
228 self.print('')
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
229 for h in self.headers:
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
230 print(h, file=output_file)
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
231 self.print('')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
232 self.print('class Parser(LRParser):')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
233 self.print(' def __init__(self):')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
234 # Generate rules:
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
235 self.print(' self.start_symbol = "{}"'.format(self.grammar.start_symbol))
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
236 self.print(' self.grammar = Grammar({})'.format(self.grammar.terminals))
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
237 for rule_number, rule in enumerate(self.grammar.productions):
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
238 rule.f_name = 'action_{}_{}'.format(rule.name, rule_number)
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
239 self.print(' self.grammar.add_production("{}", {}, self.{})'.format(rule.name, rule.symbols, rule.f_name))
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
240 # Fill action table:
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
241 self.print(' self.action_table = {}')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
242 for state in self.action_table:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
243 action = self.action_table[state]
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
244 self.print(' self.action_table[{}] = {}'.format(state, action))
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
245 self.print('')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
246
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
247 # Fill goto table:
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
248 self.print(' self.goto_table = {}')
334
6f4753202b9a Added more recipes
Windel Bouwman
parents: 323
diff changeset
249 for state_number in self.goto_table:
6f4753202b9a Added more recipes
Windel Bouwman
parents: 323
diff changeset
250 to = self.goto_table[state_number]
6f4753202b9a Added more recipes
Windel Bouwman
parents: 323
diff changeset
251 self.print(' self.goto_table[{}] = {}'.format(state_number, to))
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
252 self.print('')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
253
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
254 # Generate a function for each action:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
255 for rule in self.grammar.productions:
334
6f4753202b9a Added more recipes
Windel Bouwman
parents: 323
diff changeset
256 num_symbols = len(rule.symbols)
6f4753202b9a Added more recipes
Windel Bouwman
parents: 323
diff changeset
257 args = ', '.join('arg{}'.format(n + 1) for n in range(num_symbols))
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
258 self.print(' def {}(self, {}):'.format(rule.f_name, args))
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
259 if rule.f == None:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
260 semantics = 'pass'
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
261 else:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
262 semantics = str(rule.f)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
263 if semantics.strip() == '':
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
264 semantics = 'pass'
334
6f4753202b9a Added more recipes
Windel Bouwman
parents: 323
diff changeset
265 for n in range(num_symbols):
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
266 semantics = semantics.replace('${}'.format(n + 1), 'arg{}'.format(n + 1))
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
267 self.print(' {}'.format(semantics))
322
44f336460c2a Half of use of burg spec for arm
Windel Bouwman
parents: 321
diff changeset
268 self.print('')
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
269
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
270
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
271 def make_argument_parser():
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
272 # Parse arguments:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
273 parser = argparse.ArgumentParser(description='xacc compiler compiler')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
274 parser.add_argument('source', type=argparse.FileType('r'), \
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
275 help='the parser specification')
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
276 parser.add_argument('-o', '--output', type=argparse.FileType('w'), \
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
277 default=sys.stdout)
322
44f336460c2a Half of use of burg spec for arm
Windel Bouwman
parents: 321
diff changeset
278 return parser
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
279
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
280
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
281 def load_as_module(filename):
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
282 """ Load a parser spec file, generate LR tables and create module """
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
283 ob = io.StringIO()
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
284 args = argparse.Namespace(source=open(filename), output=ob)
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
285 main(args)
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
286
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
287 parser_mod = types.ModuleType('generated_parser')
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
288 exec(ob.getvalue(), parser_mod.__dict__)
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
289 return parser_mod
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
290
342
86b02c98a717 Moved target directory
Windel Bouwman
parents: 340
diff changeset
291
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
292 def main(args):
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
293 src = args.source.read()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
294 args.source.close()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
295
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
296 # Construction of generator parts:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
297 lexer = XaccLexer()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
298 parser = XaccParser(lexer)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
299 generator = XaccGenerator()
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
300
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
301 # Sequence source through the generator parts:
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
302 lexer.feed(src)
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
303 grammar = parser.parse_grammar()
319
8d07a4254f04 Work on burg
Windel Bouwman
parents: 318
diff changeset
304 generator.generate(grammar, parser.headers, args.output)
318
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
305
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
306
e84047f29c78 Add burg and yacc initial attempts
Windel Bouwman
parents:
diff changeset
307 if __name__ == '__main__':
321
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
308 args = make_argument_parser().parse_args()
8c569fbe60e4 Load yacc and burg dynamic
Windel Bouwman
parents: 319
diff changeset
309 main(args)